Github/nsjail
1
0
Fork 0
mirror of https://github.com/google/nsjail.git synced 2026-08-30 18:41:30 -07:00
Code Issues Packages Projects Releases Wiki Activity

Integrate a seccomp user notification to trace and log sandboxed filesystem and network operations

Browse Source 
* Introduces a dedicated, async worker thread that monitors SECCOMP_RET_USER_NOTIF events
 * Produces telemetry using protobufs to track resolved namespace paths, and network endpoints
This commit is contained in:
Robert Swiecki
2026-04-03 11:20:02 +02:00
parent 4f69b0bffb
commit ee2d05e874
19 changed files with 1440 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@@ -1,4 +1,6 @@
*.o
config.pb.h
config.pb.cc
unotify/unotify.pb.h
unotify/unotify.pb.cc
nsjail

21
Makefile
View File

@@ -61,8 +61,8 @@ ifneq ($(PASTA_BIN_PATH),)
CXXFLAGS += -DPASTA_BIN_PATH='"$(PASTA_BIN_PATH)"'
endif
SRCS_CXX = caps.cc cgroup.cc cgroup2.cc cmdline.cc config.cc contain.cc cpu.cc logs.cc mnt.cc mnt_legacy.cc mnt_newapi.cc net.cc nsjail.cc pid.cc sandbox.cc subproc.cc uts.cc user.cc util.cc nstun/nstun.cc nstun/policy.cc nstun/encap.cc nstun/iface.cc nstun/tun.cc nstun/ip.cc nstun/icmp.cc nstun/udp.cc nstun/tcp.cc
SRCS_PROTO = config.proto
SRCS_CXX = caps.cc cgroup.cc cgroup2.cc cmdline.cc config.cc contain.cc cpu.cc logs.cc mnt.cc mnt_legacy.cc mnt_newapi.cc net.cc nsjail.cc pid.cc sandbox.cc subproc.cc uts.cc user.cc unotify/unotify.cc unotify/stats.cc unotify/syscall.cc util.cc nstun/nstun.cc nstun/policy.cc nstun/encap.cc nstun/iface.cc nstun/tun.cc nstun/ip.cc nstun/icmp.cc nstun/udp.cc nstun/tcp.cc
SRCS_PROTO = config.proto unotify/unotify.proto
SRCS_PB_CXX = $(SRCS_PROTO:.proto=.pb.cc)
SRCS_PB_H = $(SRCS_PROTO:.proto=.pb.h)
@@ -256,13 +256,23 @@ mnt_legacy.o: mnt_legacy.h mnt.h nsjail.h config.pb.h logs.h macros.h util.h
mnt_newapi.o: mnt_newapi.h mnt.h nsjail.h config.pb.h logs.h util.h
net.o: net.h nsjail.h config.pb.h logs.h macros.h nstun/nstun.h util.h
nsjail.o: nsjail.h config.pb.h cgroup2.h cmdline.h logs.h macros.h net.h
nsjail.o: sandbox.h subproc.h util.h
nsjail.o: sandbox.h subproc.h unotify/unotify.h util.h
pid.o: pid.h nsjail.h config.pb.h logs.h subproc.h
sandbox.o: sandbox.h nsjail.h config.pb.h kafel/include/kafel.h logs.h util.h
sandbox.o: sandbox.h nsjail.h config.pb.h subproc.h kafel/include/kafel.h
sandbox.o: logs.h unotify/syscall_defs.h util.h
subproc.o: subproc.h nsjail.h config.pb.h cgroup.h cgroup2.h contain.h logs.h
subproc.o: macros.h net.h nstun/nstun.h sandbox.h user.h util.h
subproc.o: macros.h net.h nstun/nstun.h sandbox.h unotify/unotify.h user.h
subproc.o: util.h
uts.o: uts.h nsjail.h config.pb.h logs.h
user.o: user.h nsjail.h config.pb.h logs.h macros.h subproc.h util.h
unotify/unotify.o: unotify/unotify.h nsjail.h config.pb.h logs.h
unotify/unotify.o: unotify/record.h unotify/unotify.pb.h unotify/stats.h
unotify/unotify.o: unotify/syscall.h util.h
unotify/stats.o: unotify/stats.h nsjail.h config.pb.h unotify/record.h
unotify/stats.o: unotify/unotify.pb.h logs.h util.h
unotify/syscall.o: unotify/syscall.h unotify/record.h unotify/unotify.pb.h
unotify/syscall.o: logs.h macros.h unotify/syscall_defs.h util.h nsjail.h
unotify/syscall.o: config.pb.h
util.o: util.h nsjail.h config.pb.h logs.h macros.h
nstun/nstun.o: nstun/nstun.h nstun/core.h nstun/net_defs.h nstun/icmp.h
nstun/nstun.o: nstun/iface.h nstun/ip.h logs.h macros.h nstun/policy.h
@@ -286,3 +296,4 @@ nstun/tcp.o: nstun/tcp.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/tcp.o: nstun/encap.h logs.h macros.h nstun/policy.h nstun/tun.h util.h
nstun/tcp.o: nsjail.h config.pb.h
config.pb.o: config.pb.h
unotify/unotify.pb.o: unotify/unotify.pb.h

8
cmdline.cc
View File

@@ -144,6 +144,8 @@ static const struct custom_option custom_opts[] = {
{ { "seccomp_policy", required_argument, nullptr, 'P' }, "Path to file containing seccomp-bpf policy (see kafel/)" },
{ { "seccomp_string", required_argument, nullptr, 0x0901 }, "String with kafel seccomp-bpf policy (see kafel/)" },
{ { "seccomp_log", no_argument, nullptr, 0x0902 }, "Use SECCOMP_FILTER_FLAG_LOG. Log all actions except SECCOMP_RET_ALLOW). Supported since kernel version 4.14" },
{ { "seccomp_unotify", no_argument, nullptr, 0x0905 }, "Use SECCOMP_RET_USER_NOTIF and trace accessed files and network sockets" },
{ { "seccomp_unotify_report", required_argument, nullptr, 0x0906 }, "File to write the seccomp_unotify report to" },
{ { "nice_level", required_argument, nullptr, 0x0903 }, "Set jailed process niceness (-20 is highest -priority, 19 is lowest). By default, set to 19" },
{ { "cgroup_mem_max", required_argument, nullptr, 0x0801 }, "Maximum number of bytes to use in the group (default: '0' - disabled)" },
{ { "cgroup_mem_memsw_max", required_argument, nullptr, 0x0804 }, "Maximum number of memory+swap bytes to use (default: '0' - disabled)" },
@@ -974,6 +976,12 @@ std::unique_ptr<nsj_t> parseArgs(int argc, char* argv[]) {
case 0x902:
nsj->njc.set_seccomp_log(true);
break;
case 0x905:
nsj->njc.set_seccomp_unotify(true);
break;
case 0x906:
nsj->njc.set_seccomp_unotify_report(optarg);
break;
case 0x903:
nsj->njc.set_nice_level((int)strtol(optarg, NULL, 0));
break;

5
config.proto
View File

@@ -220,6 +220,11 @@ message NsJailConfig {
/* Setting it to true makes audit write seccomp logs to dmesg */
optional bool seccomp_log = 70 [default = false];
/* Use SECCOMP_RET_USER_NOTIF and trace accessed files and network sockets */
optional bool seccomp_unotify = 99 [default = false];
/* File to write the seccomp_unotify report to */
optional string seccomp_unotify_report = 100;
/* If > 0, maximum cumulative size of RAM used inside any jail */
optional uint64 cgroup_mem_max = 71 [default = 0]; /* In bytes */
/* If > 0, maximum cumulative size of RAM + swap used inside any jail */

4
nsjail.cc
View File

@@ -47,6 +47,7 @@
#include "net.h"
#include "sandbox.h"
#include "subproc.h"
#include "unotify/unotify.h"
#include "util.h"
namespace nsjail {
@@ -186,7 +187,7 @@ static bool pipeTraffic(nsj_t* nsj, int listenfd) {
fds[i].events &= ~POLLOUT;
}
}
for (size_t i = 0; i < fds.size() - 3; i += 3) {
for (size_t i = 0; i < nsj->pipes.size() * 3; i += 3) {
const size_t pipe_no = i / 3;
int in, out;
const char* direction;
@@ -394,6 +395,7 @@ int main(int argc, char* argv[]) {
subproc::killAndReapAll(nsj.get(), SIGKILL);
sandbox::closePolicy(nsj.get());
unotify::stop(nsj.get());
/* Try to restore the underlying console's params in case some program has changed it */
if (!nsj->njc.daemon()) {
nsjail::setTC(STDIN_FILENO, trm.get());

1
nsjail.h
View File

@@ -101,6 +101,7 @@ struct nsj_t {
bool mnt_newapi;
bool is_proc_rw;
struct sock_fprog seccomp_fprog;
struct sock_fprog seccomp_unotify_fprog;
};
#endif /* _NSJAIL_H */

152
sandbox.cc
View File

@@ -21,6 +21,7 @@
#include "sandbox.h"
#include <fcntl.h>
#include <linux/filter.h>
#include <linux/seccomp.h>
#include <stddef.h>
@@ -29,10 +30,13 @@
#include <sys/syscall.h>
#include <unistd.h>
#include "subproc.h"
extern "C" {
#include "kafel.h"
}
#include "logs.h"
#include "unotify/syscall_defs.h"
#include "util.h"
namespace sandbox {
@@ -49,45 +53,97 @@ namespace sandbox {
#define SECCOMP_FILTER_FLAG_LOG (1UL << 1)
#endif /* SECCOMP_FILTER_FLAG_LOG */
static bool prepareAndCommit(nsj_t* nsj) {
if (nsj->njc.seccomp_policy_file().empty() && nsj->njc.seccomp_string().empty()) {
#ifndef SECCOMP_FILTER_FLAG_NEW_LISTENER
#define SECCOMP_FILTER_FLAG_NEW_LISTENER (1UL << 3)
#endif /* SECCOMP_FILTER_FLAG_NEW_LISTENER */
bool installUnotifyFilter(nsj_t* nsj, int pipefd) {
if (!nsj->njc.seccomp_unotify()) {
return true;
}
if (nsj->seccomp_unotify_fprog.len == 0) {
LOG_E("seccomp_unotify enabled but no BPF program compiled");
return false;
}
if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) {
PLOG_W("prctl(PR_SET_NO_NEW_PRIVS, 1) failed");
return false;
}
if (nsj->njc.seccomp_log()) {
#ifndef __NR_seccomp
LOG_E("The __NR_seccomp is not defined with this kernel's header files (kernel "
"headers too old?)");
int ret = util::syscall(__NR_seccomp, (uintptr_t)SECCOMP_SET_MODE_FILTER,
(uintptr_t)SECCOMP_FILTER_FLAG_NEW_LISTENER, (uintptr_t)&nsj->seccomp_unotify_fprog);
if (ret == -1) {
PLOG_E("seccomp(SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_NEW_LISTENER) failed");
return false;
}
int unotif_fd = ret;
if (fcntl(unotif_fd, F_SETFD, FD_CLOEXEC) == -1) {
PLOG_E("fcntl(unotif_fd, F_SETFD, FD_CLOEXEC) failed");
close(unotif_fd);
return false;
#else
if (util::syscall(__NR_seccomp, (uintptr_t)SECCOMP_SET_MODE_FILTER,
(uintptr_t)(SECCOMP_FILTER_FLAG_TSYNC | SECCOMP_FILTER_FLAG_LOG),
(uintptr_t)&nsj->seccomp_fprog) == -1) {
PLOG_E("seccomp(SECCOMP_SET_MODE_FILTER, SECCOMP_FILTER_FLAG_TSYNC | "
"SECCOMP_FILTER_FLAG_LOG) failed");
return false;
}
return true;
#endif /* __NR_seccomp */
}
if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &nsj->seccomp_fprog, 0UL, 0UL)) {
PLOG_W("prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER) failed");
if (!util::sendFd(pipefd, unotif_fd)) {
PLOG_E("sendFd(unotif_fd) to parent failed");
close(unotif_fd);
return false;
}
LOG_D("Child: sent unotif_fd=%d to parent", unotif_fd);
close(unotif_fd);
return true;
}
static bool prepareAndCommit(nsj_t* nsj) {
if (nsj->seccomp_fprog.len == 0) {
return true;
}
/* PR_SET_NO_NEW_PRIVS is idempotent; may already be set by installUnotifyFilter */
if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0)) {
PLOG_W("prctl(PR_SET_NO_NEW_PRIVS, 1) failed");
return false;
}
unsigned int flags = 0;
if (nsj->njc.seccomp_log()) {
flags |= (SECCOMP_FILTER_FLAG_LOG | SECCOMP_FILTER_FLAG_TSYNC);
}
if (flags != 0) {
int ret = util::syscall(__NR_seccomp, (uintptr_t)SECCOMP_SET_MODE_FILTER,
(uintptr_t)flags, (uintptr_t)&nsj->seccomp_fprog);
if (ret == -1) {
PLOG_E("seccomp(SECCOMP_SET_MODE_FILTER) failed");
return false;
}
} else {
if (prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &nsj->seccomp_fprog, 0, 0)) {
PLOG_W("prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER) failed");
return false;
}
}
return true;
}
bool applyPolicy(nsj_t* nsj) {
bool applyPolicy(nsj_t* nsj, int pipefd) {
if (pipefd != -1 && nsj->njc.seccomp_unotify()) {
if (!installUnotifyFilter(nsj, pipefd)) {
return false;
}
}
return prepareAndCommit(nsj);
}
bool preparePolicy(nsj_t* nsj) {
if (nsj->njc.seccomp_policy_file().empty() && nsj->njc.seccomp_string().empty()) {
nsj->seccomp_fprog.len = 0;
nsj->seccomp_fprog.filter = NULL;
nsj->seccomp_unotify_fprog.len = 0;
nsj->seccomp_unotify_fprog.filter = NULL;
if (nsj->njc.seccomp_policy_file().empty() && nsj->njc.seccomp_string().empty() &&
!nsj->njc.seccomp_unotify()) {
return true;
}
if (!nsj->njc.seccomp_policy_file().empty() && !nsj->njc.seccomp_string().empty()) {
@@ -96,8 +152,25 @@ bool preparePolicy(nsj_t* nsj) {
return false;
}
kafel_ctxt_t ctxt = kafel_ctxt_create();
if (nsj->njc.seccomp_unotify()) {
kafel_ctxt_t unotify_ctxt = kafel_ctxt_create();
std::string unotify_policy = unotify::buildKafelPolicy();
kafel_set_input_string(unotify_ctxt, unotify_policy.c_str());
if (kafel_compile(unotify_ctxt, &nsj->seccomp_unotify_fprog) != 0) {
LOG_E("Could not compile the default unotify seccomp policy: %s",
kafel_error_msg(unotify_ctxt));
kafel_ctxt_destroy(&unotify_ctxt);
return false;
}
kafel_ctxt_destroy(&unotify_ctxt);
}
if (nsj->njc.seccomp_policy_file().empty() && nsj->njc.seccomp_string().empty()) {
return true;
}
kafel_ctxt_t ctxt = kafel_ctxt_create();
std::string combined_seccomp_policy;
if (!nsj->njc.seccomp_policy_file().empty()) {
FILE* f = fopen(nsj->njc.seccomp_policy_file().c_str(), "r");
if (!f) {
@@ -109,34 +182,39 @@ bool preparePolicy(nsj_t* nsj) {
LOG_D("Compiling seccomp policy from file: '%s'",
nsj->njc.seccomp_policy_file().c_str());
kafel_set_input_file(ctxt, f);
}
std::string combined_seccomp_policy;
for (const auto& s : nsj->njc.seccomp_string()) {
combined_seccomp_policy += s;
combined_seccomp_policy += '\n';
}
if (!combined_seccomp_policy.empty()) {
LOG_D(
"Compiling seccomp policy from string: '%s'", combined_seccomp_policy.c_str());
kafel_set_input_string(ctxt, combined_seccomp_policy.c_str());
} else {
for (const auto& s : nsj->njc.seccomp_string()) {
combined_seccomp_policy += s;
combined_seccomp_policy += '\n';
}
if (!combined_seccomp_policy.empty()) {
LOG_D("Compiling seccomp policy from string:\n%s",
combined_seccomp_policy.c_str());
kafel_set_input_string(ctxt, combined_seccomp_policy.c_str());
}
}
if (kafel_compile(ctxt, &nsj->seccomp_fprog) != 0) {
LOG_W("Could not compile policy: %s", kafel_error_msg(ctxt));
LOG_E("Could not compile policy: %s", kafel_error_msg(ctxt));
kafel_ctxt_destroy(&ctxt);
return false;
}
kafel_ctxt_destroy(&ctxt);
return true;
}
void closePolicy(nsj_t* nsj) {
if (!nsj->seccomp_fprog.filter) {
return;
if (nsj->seccomp_fprog.filter) {
free(nsj->seccomp_fprog.filter);
nsj->seccomp_fprog.filter = nullptr;
nsj->seccomp_fprog.len = 0;
}
if (nsj->seccomp_unotify_fprog.filter) {
free(nsj->seccomp_unotify_fprog.filter);
nsj->seccomp_unotify_fprog.filter = nullptr;
nsj->seccomp_unotify_fprog.len = 0;
}
free(nsj->seccomp_fprog.filter);
nsj->seccomp_fprog.filter = nullptr;
nsj->seccomp_fprog.len = 0;
}
} // namespace sandbox

3
sandbox.h
View File

@@ -28,7 +28,8 @@
namespace sandbox {
bool applyPolicy(nsj_t* nsj);
bool installUnotifyFilter(nsj_t* nsj, int pipefd);
bool applyPolicy(nsj_t* nsj, int pipefd);
bool preparePolicy(nsj_t* nsj);
void closePolicy(nsj_t* nsj);

61
subproc.cc
View File

@@ -52,6 +52,7 @@
#include "net.h"
#include "nstun/nstun.h"
#include "sandbox.h"
#include "unotify/unotify.h"
#include "user.h"
#include "util.h"
@@ -145,7 +146,6 @@ static bool resetEnv(void) {
return true;
}
static const char kSubprocDoneChar = 'D';
static const char kSubprocErrorChar = 'E';
static const std::string concatArgs(const std::vector<const char*>& argv) {
@@ -196,14 +196,14 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
if (!net::initChildPreSync(nsj, pipefd)) {
return;
}
}
char doneChar;
if (util::readFromFd(pipefd, &doneChar, sizeof(doneChar)) != sizeof(doneChar)) {
return;
}
if (doneChar != kSubprocDoneChar) {
return;
char doneChar;
if (util::readFromFd(pipefd, &doneChar, sizeof(doneChar)) !=
sizeof(doneChar)) {
return;
}
if (doneChar != kSubprocDoneChar) {
return;
}
}
}
if (!contain::containProc(nsj)) {
@@ -228,10 +228,20 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
LOG_D("Exec: %s, Args: [%s]", QC(nsj->njc.exec_bin().path()), concatArgs(argv).c_str());
/* Should be the last one in the sequence */
if (!sandbox::applyPolicy(nsj)) {
if (!sandbox::applyPolicy(nsj, pipefd)) {
return;
}
if (pipefd != -1 && nsj->njc.seccomp_unotify()) {
char doneChar;
if (util::readFromFd(pipefd, &doneChar, sizeof(doneChar)) != sizeof(doneChar)) {
return;
}
if (doneChar != kSubprocDoneChar) {
return;
}
}
if (nsj->njc.exec_bin().exec_fd()) {
#if defined(__NR_execveat)
util::syscall(__NR_execveat, nsj->exec_fd, (uintptr_t)"", (uintptr_t)argv.data(),
@@ -477,6 +487,25 @@ static bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
LOG_E("Couldn't signal the new process via a socketpair");
return false;
}
if (nsj->njc.seccomp_unotify()) {
int unotif_fd = util::recvFd(pipefd);
if (unotif_fd == -1) {
LOG_E("Failed to receive unotif_fd from child");
return false;
}
LOG_D("Received unotif_fd=%d from child", unotif_fd);
if (!unotify::start(nsj, unotif_fd)) {
LOG_E("Failed to start unotify thread");
close(unotif_fd);
return false;
}
if (!util::writeToFd(pipefd, &kSubprocDoneChar, sizeof(kSubprocDoneChar))) {
LOG_E("Couldn't signal the new process via a socketpair");
return false;
}
}
return true;
}
@@ -537,11 +566,13 @@ pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) {
}
char rcvChar;
if (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar) &&
rcvChar == kSubprocErrorChar) {
LOG_W("Received error message from the child process before it has been executed");
close(parent_fd);
return -1;
while (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) {
if (rcvChar == kSubprocErrorChar) {
LOG_W("Received error message from the child process before it has been "
"executed");
close(parent_fd);
return -1;
}
}
close(parent_fd);

1
subproc.h
View File

@@ -32,6 +32,7 @@
#include "nsjail.h"
namespace subproc {
constexpr char kSubprocDoneChar = 'D';
/* 0 - network connection limit reached, -1 - error */
pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err);

62
unotify/record.h Normal file
View File

@@ -0,0 +1,62 @@
#ifndef NSJAIL_UNOTIFY_RECORD_H
#define NSJAIL_UNOTIFY_RECORD_H
#include <string>
#include <tuple>
#include <vector>
#include "unotify/unotify.pb.h"
namespace unotify {
struct PathInfoRecord {
std::string path;
Stat_Path_Type jail_type = Stat_Path_Type_NONEXISTENT;
Stat_Path_Type main_type = Stat_Path_Type_NONEXISTENT;
Stat_Path_Mode mode = Stat_Path_Mode_UNSPECIFIED;
std::string mode_extra;
/* Lexicographical comparison to support std::map usage in stats.cc */
bool operator<(const PathInfoRecord& o) const {
return std::tie(path, jail_type, main_type, mode, mode_extra) <
std::tie(o.path, o.jail_type, o.main_type, o.mode, o.mode_extra);
}
};
struct ResourceRecord {
bool has_path1 = false;
PathInfoRecord path1;
bool has_path2 = false;
PathInfoRecord path2;
bool has_net = false;
Stat_NetResource_Type net_type = Stat_NetResource_Type_UNKNOWN;
std::string net_endpoint;
bool has_net_port = false;
uint32_t net_port = 0;
bool has_net_path = false;
PathInfoRecord net_path;
/* Lexicographical comparison to support std::map usage in stats.cc */
bool operator<(const ResourceRecord& o) const {
return std::tie(has_path1, path1, has_path2, path2, has_net, net_type, net_endpoint, has_net_port, net_port, has_net_path, net_path) <
std::tie(o.has_path1, o.path1, o.has_path2, o.path2, o.has_net, o.net_type, o.net_endpoint, o.has_net_port, o.net_port, o.has_net_path, o.net_path);
}
};
struct SyscallRecord {
std::string name;
std::vector<std::string> args;
ResourceRecord res;
/* Lexicographical comparison to support std::map usage in stats.cc */
bool operator<(const SyscallRecord& other) const {
return std::tie(name, args, res) <
std::tie(other.name, other.args, other.res);
}
};
} // namespace unotify
#endif /* NSJAIL_UNOTIFY_RECORD_H */

186
unotify/stats.cc Normal file
View File

@@ -0,0 +1,186 @@
#include "unotify/stats.h"
#include <fcntl.h>
#include <google/protobuf/text_format.h>
#include <unistd.h>
#include <map>
#include <mutex>
#include "logs.h"
#include "util.h"
namespace unotify {
static std::mutex stats_mu;
/* Aggregates syscall statistics on the fly.
* Uses std::map which requires SyscallRecord to have operator<. */
static std::map<SyscallRecord, size_t> stats;
void addStat(const SyscallRecord& rec) {
std::lock_guard<std::mutex> lock(stats_mu);
stats[rec]++;
}
static void fillPathInfoPb(Stat_Path* pb, const PathInfoRecord& rec) {
pb->set_path(rec.path);
pb->set_jail_type(rec.jail_type);
pb->set_main_type(rec.main_type);
pb->set_exists_in_jail(
rec.jail_type != Stat_Path_Type_NONEXISTENT && rec.jail_type != Stat_Path_Type_UNKNOWN);
pb->set_exists_in_main(
rec.main_type != Stat_Path_Type_NONEXISTENT && rec.main_type != Stat_Path_Type_UNKNOWN);
if (rec.mode != Stat_Path_Mode_UNSPECIFIED) {
pb->set_mode(rec.mode);
}
}
struct NetInfoRecord {
Stat_NetResource_Type type = Stat_NetResource_Type_UNKNOWN;
std::string endpoint;
bool has_port = false;
uint32_t port = 0;
bool has_path = false;
PathInfoRecord path;
bool operator<(const NetInfoRecord& o) const {
if (type != o.type) return type < o.type;
if (endpoint != o.endpoint) return endpoint < o.endpoint;
if (has_port != o.has_port) return has_port < o.has_port;
if (has_port) {
if (port != o.port) return port < o.port;
}
if (has_path != o.has_path) return has_path < o.has_path;
if (has_path) {
if (path < o.path) return true;
if (o.path < path) return false;
}
return false;
}
};
struct SyscallKey {
std::string name;
std::vector<std::string> args;
bool operator<(const SyscallKey& o) const {
if (name != o.name) return name < o.name;
return args < o.args;
}
};
struct FsStats {
uint64_t count = 0;
std::map<SyscallKey, uint64_t> syscalls;
};
struct NetStats {
uint64_t count = 0;
std::map<SyscallKey, uint64_t> syscalls;
};
void printStats(nsj_t* nsj) {
if (!nsj->njc.seccomp_unotify()) {
return;
}
if (stats.empty()) {
return; // Do not emit if empty
}
std::map<PathInfoRecord, FsStats> fs_stats;
std::map<NetInfoRecord, NetStats> net_stats;
{
std::lock_guard<std::mutex> lock(stats_mu);
for (const auto& [rec, count] : stats) {
SyscallKey sys_key{rec.name, rec.args};
if (rec.res.has_path1) {
SyscallKey p1_key = sys_key;
if (!rec.res.path1.mode_extra.empty()) {
p1_key.args.push_back(
"mode_extra=" + rec.res.path1.mode_extra);
}
fs_stats[rec.res.path1].count += count;
fs_stats[rec.res.path1].syscalls[p1_key] += count;
}
if (rec.res.has_path2) {
SyscallKey p2_key = sys_key;
if (!rec.res.path2.mode_extra.empty()) {
p2_key.args.push_back(
"mode_extra=" + rec.res.path2.mode_extra);
}
fs_stats[rec.res.path2].count += count;
fs_stats[rec.res.path2].syscalls[p2_key] += count;
}
if (rec.res.has_net) {
NetInfoRecord net_rec;
net_rec.type = rec.res.net_type;
net_rec.endpoint = rec.res.net_endpoint;
net_rec.has_port = rec.res.has_net_port;
net_rec.port = rec.res.net_port;
net_rec.has_path = rec.res.has_net_path;
net_rec.path = rec.res.net_path;
net_stats[net_rec].count += count;
net_stats[net_rec].syscalls[sys_key] += count;
}
}
}
Stat report_pb;
for (const auto& [path_rec, fs_stat] : fs_stats) {
Stat_Path* fs_pb = report_pb.add_fs_access();
fs_pb->set_count(fs_stat.count);
fillPathInfoPb(fs_pb, path_rec);
for (const auto& [sys_key, sys_count] : fs_stat.syscalls) {
Stat_Syscall* sys_pb = fs_pb->add_syscall();
sys_pb->set_name(sys_key.name);
sys_pb->set_count(sys_count);
for (const auto& arg : sys_key.args) {
sys_pb->add_args(arg);
}
}
}
for (const auto& [net_rec, net_stat] : net_stats) {
Stat_NetResource* net_pb = report_pb.add_net_access();
net_pb->set_count(net_stat.count);
net_pb->set_type(net_rec.type);
if (!net_rec.endpoint.empty()) {
net_pb->set_endpoint(net_rec.endpoint);
}
if (net_rec.has_port) {
net_pb->set_port(net_rec.port);
}
if (net_rec.has_path) {
fillPathInfoPb(net_pb->mutable_socket_path(), net_rec.path);
}
for (const auto& [sys_key, sys_count] : net_stat.syscalls) {
Stat_Syscall* sys_pb = net_pb->add_syscall();
sys_pb->set_name(sys_key.name);
sys_pb->set_count(sys_count);
for (const auto& arg : sys_key.args) {
sys_pb->add_args(arg);
}
}
}
std::string text_report;
google::protobuf::TextFormat::PrintToString(report_pb, &text_report);
LOG_I("unotify report:\n%s", text_report.c_str());
if (!nsj->njc.seccomp_unotify_report().empty()) {
if (!util::writeBufToFile(nsj->njc.seccomp_unotify_report().c_str(),
text_report.data(), text_report.size(), O_CREAT | O_WRONLY | O_TRUNC)) {
PLOG_W("Failed to write unotify report to %s",
nsj->njc.seccomp_unotify_report().c_str());
}
}
}
} // namespace unotify

14
unotify/stats.h Normal file
View File

@@ -0,0 +1,14 @@
#ifndef NSJAIL_UNOTIFY_STATS_H
#define NSJAIL_UNOTIFY_STATS_H
#include "nsjail.h"
#include "unotify/record.h"
namespace unotify {
void addStat(const SyscallRecord& rec);
void printStats(nsj_t* nsj);
} // namespace unotify
#endif /* NSJAIL_UNOTIFY_STATS_H */

580
unotify/syscall.cc Normal file
View File

@@ -0,0 +1,580 @@
#include "unotify/syscall.h"
#include <arpa/inet.h>
#include <fcntl.h>
#include <linux/audit.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <unistd.h>
#include <string>
#include <vector>
#include "logs.h"
#include "macros.h"
#include "unotify/syscall_defs.h"
#include "util.h"
namespace unotify {
constexpr size_t kMaxArgs = 128;
constexpr size_t kMaxPathLen = 4096;
/* Helper functions string/memory reading, path resolution, etc. */
static std::string getSocketType(pid_t pid, int target_fd) {
int pidfd = syscall(__NR_pidfd_open, pid, 0);
if (pidfd < 0) {
return "";
}
int local_fd = syscall(__NR_pidfd_getfd, pidfd, target_fd, 0);
close(pidfd);
if (local_fd < 0) {
return "";
}
int type = 0, proto = 0;
socklen_t len = sizeof(type);
std::string type_str;
if (getsockopt(local_fd, SOL_SOCKET, SO_TYPE, &type, &len) == 0) {
struct {
const int val;
const char* const name;
} static const sockTypes[] = {
NS_VALSTR_STRUCT(SOCK_STREAM),
NS_VALSTR_STRUCT(SOCK_DGRAM),
NS_VALSTR_STRUCT(SOCK_RAW),
NS_VALSTR_STRUCT(SOCK_SEQPACKET),
};
bool found = false;
for (const auto& i : sockTypes) {
if (type == i.val) {
type_str = i.name;
found = true;
break;
}
}
if (!found) type_str = "SOCK_TYPE_" + std::to_string(type);
}
len = sizeof(proto);
if (getsockopt(local_fd, SOL_SOCKET, SO_PROTOCOL, &proto, &len) == 0) {
struct {
const int val;
const char* const name;
} static const protoTypes[] = {
NS_VALSTR_STRUCT(IPPROTO_TCP),
NS_VALSTR_STRUCT(IPPROTO_UDP),
NS_VALSTR_STRUCT(IPPROTO_ICMP),
NS_VALSTR_STRUCT(IPPROTO_ICMPV6),
};
bool found = false;
for (const auto& i : protoTypes) {
if (proto == i.val) {
type_str += " (";
type_str += i.name;
type_str += ")";
found = true;
break;
}
}
if (!found && proto != 0) type_str += " (proto_" + std::to_string(proto) + ")";
}
close(local_fd);
return type_str;
}
static std::string readStringFromMem(pid_t pid, uint64_t addr) {
if (addr == 0) return "NULL";
char buf[kMaxPathLen];
struct iovec local = {buf, sizeof(buf) - 1};
struct iovec remote = {(void*)addr, sizeof(buf) - 1};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) {
return "<invalid_ptr>";
}
buf[ret] = '\0';
return std::string(buf);
}
static std::vector<std::string> readStringArrayFromMem(pid_t pid, uint64_t addr, bool is_32bit) {
std::vector<std::string> arr;
if (addr == 0) return arr;
if (is_32bit) {
uint32_t ptrs[kMaxArgs];
struct iovec local = {ptrs, sizeof(ptrs)};
struct iovec remote = {(void*)addr, sizeof(ptrs)};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) return arr;
int num_ptrs = ret / sizeof(uint32_t);
for (int i = 0; i < num_ptrs; i++) {
if (ptrs[i] == 0) break;
arr.push_back(readStringFromMem(pid, ptrs[i]));
}
} else {
uint64_t ptrs[kMaxArgs];
struct iovec local = {ptrs, sizeof(ptrs)};
struct iovec remote = {(void*)addr, sizeof(ptrs)};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) return arr;
int num_ptrs = ret / sizeof(uint64_t);
for (int i = 0; i < num_ptrs; i++) {
if (ptrs[i] == 0) break;
arr.push_back(readStringFromMem(pid, ptrs[i]));
}
}
return arr;
}
static std::string getAbsPath(pid_t pid, int dirfd, const std::string& raw_path) {
if (raw_path.empty() || raw_path[0] == '/') return raw_path;
std::string link_path;
if (dirfd == AT_FDCWD || dirfd == -100) {
link_path = "/proc/" + std::to_string(pid) + "/cwd";
} else {
link_path = "/proc/" + std::to_string(pid) + "/fd/" + std::to_string(dirfd);
}
char buf[PATH_MAX];
ssize_t len = readlink(link_path.c_str(), buf, sizeof(buf) - 1);
if (len <= 0) return raw_path; // fallback
buf[len] = '\0';
std::string abs_path = buf;
if (abs_path.back() != '/') abs_path += '/';
abs_path += raw_path;
return abs_path;
}
static void getFileMode(int flags, PathInfoRecord* out) {
if ((flags & O_ACCMODE) == O_RDONLY)
out->mode = Stat_Path_Mode_RDONLY;
else if ((flags & O_ACCMODE) == O_WRONLY)
out->mode = Stat_Path_Mode_WRONLY;
else if ((flags & O_ACCMODE) == O_RDWR)
out->mode = Stat_Path_Mode_RDWR;
else
out->mode = Stat_Path_Mode_UNSPECIFIED;
std::string mode_extra;
struct {
const int val;
const char* const name;
} static const openFlags[] = {
NS_VALSTR_STRUCT(O_CREAT),
NS_VALSTR_STRUCT(O_EXCL),
NS_VALSTR_STRUCT(O_NOCTTY),
NS_VALSTR_STRUCT(O_TRUNC),
NS_VALSTR_STRUCT(O_APPEND),
NS_VALSTR_STRUCT(O_NONBLOCK),
NS_VALSTR_STRUCT(O_DSYNC),
NS_VALSTR_STRUCT(FASYNC),
NS_VALSTR_STRUCT(O_DIRECT),
NS_VALSTR_STRUCT(O_LARGEFILE),
NS_VALSTR_STRUCT(O_DIRECTORY),
NS_VALSTR_STRUCT(O_NOFOLLOW),
NS_VALSTR_STRUCT(O_NOATIME),
NS_VALSTR_STRUCT(O_CLOEXEC),
NS_VALSTR_STRUCT(O_SYNC),
NS_VALSTR_STRUCT(O_PATH),
NS_VALSTR_STRUCT(O_TMPFILE),
};
for (const auto& i : openFlags) {
if (flags & i.val) {
mode_extra += "|";
mode_extra += i.name;
}
}
if (!mode_extra.empty()) {
out->mode_extra = mode_extra.substr(1); // remove leading pipe
}
}
static std::string getAccessMode(int mode) {
if (mode == F_OK) return "F_OK";
std::string acc;
if (mode & R_OK) acc += "R_OK|";
if (mode & W_OK) acc += "W_OK|";
if (mode & X_OK) acc += "X_OK|";
if (!acc.empty()) acc.pop_back();
return acc;
}
static Stat_Path_Type getStatInfo(const std::string& path) {
struct stat st;
if (lstat(path.c_str(), &st) == -1) {
return Stat_Path_Type_NONEXISTENT;
}
if (S_ISREG(st.st_mode)) return Stat_Path_Type_REGULAR;
if (S_ISDIR(st.st_mode)) return Stat_Path_Type_DIR;
if (S_ISCHR(st.st_mode)) return Stat_Path_Type_CHR;
if (S_ISBLK(st.st_mode)) return Stat_Path_Type_BLK;
if (S_ISFIFO(st.st_mode)) return Stat_Path_Type_FIFO;
if (S_ISLNK(st.st_mode)) return Stat_Path_Type_LINK;
if (S_ISSOCK(st.st_mode)) return Stat_Path_Type_SOCK;
return Stat_Path_Type_UNKNOWN;
}
static void populatePathInfo(pid_t pid, int dirfd, uint64_t addr, PathInfoRecord* out_rec) {
std::string raw_path = readStringFromMem(pid, addr);
out_rec->path = getAbsPath(pid, dirfd, raw_path);
out_rec->jail_type = getStatInfo("/proc/" + std::to_string(pid) + "/root" + out_rec->path);
out_rec->main_type = getStatInfo(out_rec->path);
}
/* sockaddr decoder */
static void parseSockaddr(struct seccomp_notif* req, SyscallRecord* rec, uint64_t addr,
socklen_t addrlen, const std::string& socket_type_str) {
if (addrlen > sizeof(struct sockaddr_storage) || addr == 0) return;
struct sockaddr_storage ss = {};
struct iovec local = {&ss, addrlen};
struct iovec remote = {(void*)addr, addrlen};
ssize_t read_bytes = process_vm_readv(req->pid, &local, 1, &remote, 1, 0);
if (read_bytes >= (ssize_t)sizeof(sa_family_t)) {
char host[INET6_ADDRSTRLEN] = "unknown";
int port = 0;
if (ss.ss_family == AF_INET) {
struct sockaddr_in* sin = (struct sockaddr_in*)&ss;
inet_ntop(AF_INET, &sin->sin_addr, host, sizeof(host));
port = ntohs(sin->sin_port);
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_IPV4;
rec->res.net_endpoint = std::string(host);
if (socket_type_str.find("SOCK_STREAM") != std::string::npos ||
socket_type_str.find("SOCK_DGRAM") != std::string::npos) {
rec->res.has_net_port = true;
rec->res.net_port = port;
}
} else if (ss.ss_family == AF_INET6) {
struct sockaddr_in6* sin6 = (struct sockaddr_in6*)&ss;
inet_ntop(AF_INET6, &sin6->sin6_addr, host, sizeof(host));
port = ntohs(sin6->sin6_port);
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_IPV6;
rec->res.net_endpoint = std::string(host);
if (socket_type_str.find("SOCK_STREAM") != std::string::npos ||
socket_type_str.find("SOCK_DGRAM") != std::string::npos) {
rec->res.has_net_port = true;
rec->res.net_port = port;
}
} else if (ss.ss_family == AF_UNIX) {
struct sockaddr_un* sun = (struct sockaddr_un*)&ss;
size_t path_len = 0;
if (addrlen > sizeof(sa_family_t)) {
path_len = addrlen - sizeof(sa_family_t);
}
if (path_len > sizeof(sun->sun_path)) {
path_len = sizeof(sun->sun_path);
}
/* Ensure NUL termination for non-abstract paths */
sun->sun_path[sizeof(sun->sun_path) - 1] = '\0';
std::string raw_path(sun->sun_path, strnlen(sun->sun_path, path_len));
if (path_len == 0) {
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_UNIX;
rec->res.net_endpoint = "anonymous unix socket";
} else if (sun->sun_path[0] != '\0') {
std::string abs_path = getAbsPath(req->pid, AT_FDCWD, raw_path);
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_UNIX;
rec->res.net_endpoint = abs_path;
/* Also populate FS tracking for UNIX socket paths */
rec->res.has_net_path = true;
rec->res.net_path.path = abs_path;
rec->res.net_path.jail_type = getStatInfo(
"/proc/" + std::to_string(req->pid) + "/root" + abs_path);
rec->res.net_path.main_type = getStatInfo(abs_path);
} else {
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_UNIX;
rec->res.net_endpoint =
"@" + std::string(sun->sun_path + 1, path_len - 1);
}
} else if (ss.ss_family == AF_NETLINK) {
rec->res.has_net = true;
rec->res.net_type = Stat_NetResource_Type_NETLINK;
rec->res.net_endpoint = "NETLINK";
}
}
}
/* String formatters for socket domain, type, protocol */
static std::string getDomainStr(int domain) {
struct {
const int val;
const char* const name;
} static const domains[] = {
NS_VALSTR_STRUCT(AF_UNIX),
NS_VALSTR_STRUCT(AF_LOCAL),
NS_VALSTR_STRUCT(AF_INET),
NS_VALSTR_STRUCT(AF_INET6),
NS_VALSTR_STRUCT(AF_IPX),
NS_VALSTR_STRUCT(AF_NETLINK),
NS_VALSTR_STRUCT(AF_X25),
NS_VALSTR_STRUCT(AF_AX25),
NS_VALSTR_STRUCT(AF_ATMPVC),
NS_VALSTR_STRUCT(AF_APPLETALK),
NS_VALSTR_STRUCT(AF_PACKET),
NS_VALSTR_STRUCT(AF_ALG),
NS_VALSTR_STRUCT(AF_VSOCK),
};
for (const auto& i : domains) {
if (domain == i.val) return i.name;
}
return std::to_string(domain);
}
static std::string getTypeStr(int type) {
int base_type = type & 0xf;
std::string res;
struct {
const int val;
const char* const name;
} static const types[] = {
NS_VALSTR_STRUCT(SOCK_STREAM),
NS_VALSTR_STRUCT(SOCK_DGRAM),
NS_VALSTR_STRUCT(SOCK_RAW),
NS_VALSTR_STRUCT(SOCK_RDM),
NS_VALSTR_STRUCT(SOCK_SEQPACKET),
NS_VALSTR_STRUCT(SOCK_DCCP),
NS_VALSTR_STRUCT(SOCK_PACKET),
};
bool found = false;
for (const auto& i : types) {
if (base_type == i.val) {
res = i.name;
found = true;
break;
}
}
if (!found) res = std::to_string(base_type);
if (type & SOCK_CLOEXEC) res += "|SOCK_CLOEXEC";
if (type & SOCK_NONBLOCK) res += "|SOCK_NONBLOCK";
return res;
}
static std::string getProtocolStr(int proto) {
if (proto == 0) return "0";
struct {
const int val;
const char* const name;
} static const protos[] = {
NS_VALSTR_STRUCT(IPPROTO_IP),
NS_VALSTR_STRUCT(IPPROTO_ICMP),
NS_VALSTR_STRUCT(IPPROTO_IGMP),
NS_VALSTR_STRUCT(IPPROTO_IPIP),
NS_VALSTR_STRUCT(IPPROTO_TCP),
NS_VALSTR_STRUCT(IPPROTO_EGP),
NS_VALSTR_STRUCT(IPPROTO_PUP),
NS_VALSTR_STRUCT(IPPROTO_UDP),
NS_VALSTR_STRUCT(IPPROTO_IDP),
NS_VALSTR_STRUCT(IPPROTO_TP),
NS_VALSTR_STRUCT(IPPROTO_DCCP),
NS_VALSTR_STRUCT(IPPROTO_IPV6),
NS_VALSTR_STRUCT(IPPROTO_RSVP),
NS_VALSTR_STRUCT(IPPROTO_GRE),
NS_VALSTR_STRUCT(IPPROTO_ESP),
NS_VALSTR_STRUCT(IPPROTO_AH),
NS_VALSTR_STRUCT(IPPROTO_MTP),
NS_VALSTR_STRUCT(IPPROTO_BEETPH),
NS_VALSTR_STRUCT(IPPROTO_ENCAP),
NS_VALSTR_STRUCT(IPPROTO_PIM),
NS_VALSTR_STRUCT(IPPROTO_COMP),
NS_VALSTR_STRUCT(IPPROTO_SCTP),
NS_VALSTR_STRUCT(IPPROTO_UDPLITE),
NS_VALSTR_STRUCT(IPPROTO_MPLS),
NS_VALSTR_STRUCT(IPPROTO_RAW),
NS_VALSTR_STRUCT(IPPROTO_ICMPV6),
};
for (const auto& i : protos) {
if (proto == i.val) return i.name;
}
return std::to_string(proto);
}
/* Generic arg decoder driven by ArgRole metadata from the table */
static void decodeSyscallArgs(
struct seccomp_notif* req, SyscallRecord* rec, const SyscallDef& def) {
__u64* args = req->data.args;
int current_dirfd = AT_FDCWD;
PathInfoRecord* last_path = nullptr;
std::string last_socket_type;
bool is_32bit = false;
#ifdef AUDIT_ARCH_I386
if (req->data.arch == AUDIT_ARCH_I386) is_32bit = true;
#endif
#ifdef AUDIT_ARCH_ARM
if (req->data.arch == AUDIT_ARCH_ARM) is_32bit = true;
#endif
for (int i = 0; i < 6; i++) {
__u64 arg = args[i];
switch (def.args[i]) {
case ArgRole::SKIP:
break;
case ArgRole::PATH: {
PathInfoRecord* target;
if (!rec->res.has_path1) {
rec->res.has_path1 = true;
target = &rec->res.path1;
} else {
rec->res.has_path2 = true;
target = &rec->res.path2;
}
populatePathInfo(req->pid, current_dirfd, arg, target);
last_path = target;
current_dirfd = AT_FDCWD;
break;
}
case ArgRole::DIRFD: {
int dirfd = (int)arg;
rec->args.push_back("dirfd=" + (dirfd == AT_FDCWD || dirfd == -100
? std::string("AT_FDCWD")
: std::to_string(dirfd)));
current_dirfd = dirfd;
break;
}
case ArgRole::FLAGS:
if (last_path) {
getFileMode((int)arg, last_path);
}
break;
case ArgRole::OCTAL: {
char buf[32];
snprintf(buf, sizeof(buf), "0%o", (unsigned int)arg);
rec->args.push_back("mode=" + std::string(buf));
break;
}
case ArgRole::ACCESS:
rec->args.push_back("mode=" + getAccessMode((int)arg));
break;
case ArgRole::UID:
rec->args.push_back("owner=" + std::to_string((int)arg));
break;
case ArgRole::GID:
rec->args.push_back("group=" + std::to_string((int)arg));
break;
case ArgRole::ARGV: {
auto argv = readStringArrayFromMem(req->pid, arg, is_32bit);
for (size_t j = 0; j < argv.size(); j++) {
rec->args.push_back("argv[" + std::to_string(j) + "]=" + argv[j]);
}
break;
}
case ArgRole::ENVP: {
auto envp = readStringArrayFromMem(req->pid, arg, is_32bit);
for (size_t j = 0; j < envp.size(); j++) {
rec->args.push_back("envp[" + std::to_string(j) + "]=" + envp[j]);
}
break;
}
case ArgRole::FD: {
rec->args.push_back("fd=" + std::to_string((int)arg));
last_socket_type = getSocketType(req->pid, (int)arg);
if (!last_socket_type.empty()) {
rec->args.push_back("type=" + last_socket_type);
}
break;
}
case ArgRole::SADDR: {
if (arg == 0) break;
socklen_t addrlen = (i + 1 < 6) ? (socklen_t)args[i + 1] : 0;
parseSockaddr(req, rec, arg, addrlen, last_socket_type);
break;
}
case ArgRole::ALEN:
break; /* consumed by preceding SADDR */
case ArgRole::IFLAGS:
rec->args.push_back("flags=" + std::to_string((int)arg));
break;
case ArgRole::DOMAIN:
rec->args.push_back("domain=" + getDomainStr((int)arg));
break;
case ArgRole::STYPE:
rec->args.push_back("type=" + getTypeStr((int)arg));
break;
case ArgRole::PROTO:
rec->args.push_back("protocol=" + getProtocolStr((int)arg));
break;
case ArgRole::OHOW: {
struct {
__u64 flags;
__u64 mode;
__u64 resolve;
} how = {};
struct iovec local = {&how, sizeof(how)};
struct iovec remote = {(void*)arg, sizeof(how)};
if (process_vm_readv(req->pid, &local, 1, &remote, 1, 0) >=
(ssize_t)sizeof(how.flags)) {
if (last_path) {
getFileMode((int)how.flags, last_path);
}
if (how.resolve != 0) {
rec->args.push_back(
"resolve=" + std::to_string(how.resolve));
}
}
break;
}
}
}
}
/* Public API table-driven syscall name lookup + arg decoding */
void parseSyscall(struct seccomp_notif* req, SyscallRecord* rec) {
int nr = req->data.nr;
const SyscallDef* def = nullptr;
for (size_t i = 0; i < kTracedSyscallCount; i++) {
if (kTracedSyscalls[i].nr == nr) {
rec->name = kTracedSyscalls[i].display_name;
def = &kTracedSyscalls[i];
break;
}
}
if (rec->name.empty()) {
rec->name = "sys_" + std::to_string(nr);
}
if (def) {
decodeSyscallArgs(req, rec, *def);
}
}
} // namespace unotify

14
unotify/syscall.h Normal file
View File

@@ -0,0 +1,14 @@
#ifndef NSJAIL_UNOTIFY_SYSCALL_H
#define NSJAIL_UNOTIFY_SYSCALL_H
#include <linux/seccomp.h>
#include "unotify/record.h"
namespace unotify {
void parseSyscall(struct seccomp_notif* req, SyscallRecord* rec);
} // namespace unotify
#endif /* NSJAIL_UNOTIFY_SYSCALL_H */

173
unotify/syscall_defs.h Normal file
View File

@@ -0,0 +1,173 @@
#ifndef NSJAIL_UNOTIFY_SYSCALL_DEFS_H
#define NSJAIL_UNOTIFY_SYSCALL_DEFS_H
#include <cstddef>
#include <cstdint>
#include <string>
#include <sys/syscall.h>
namespace unotify {
enum class SyscallCategory : uint8_t {
FS,
NET,
};
/*
* Describes the role of each syscall argument for automatic decoding.
*
* The generic decoder processes args[0..5] left-to-right, maintaining:
* current_dirfd - set by DIRFD, consumed+reset by PATH
* last_socket_type - set by FD, consumed by SADDR
* last_path - set by PATH, consumed by FLAGS/OHOW
*/
enum class ArgRole : uint8_t {
SKIP, /* Don't decode this argument */
PATH, /* String ptr -> populatePathInfo (alternates path1/path2) */
DIRFD, /* Directory fd; sets context for next PATH */
FLAGS, /* Open flags -> getFileMode on last populated path */
OCTAL, /* Permission mode -> "mode=0NNN" */
ACCESS, /* Access mode -> "mode=R_OK|W_OK|X_OK" */
UID, /* User id -> "owner=N" */
GID, /* Group id -> "group=N" */
ARGV, /* String array ptr -> "argv[i]=..." */
ENVP, /* String array ptr -> "envp[i]=..." */
FD, /* Socket fd -> "fd=N" + getSocketType */
SADDR, /* sockaddr ptr; next arg must be ALEN */
ALEN, /* sockaddr length; consumed by preceding SADDR */
IFLAGS, /* Integer flags -> "flags=N" */
DOMAIN, /* Socket domain -> getDomainStr */
STYPE, /* Socket type -> getTypeStr */
PROTO, /* Protocol -> getProtocolStr */
OHOW, /* struct open_how ptr -> decode flags + resolve */
};
struct SyscallDef {
int nr; /* __NR_xxx */
const char* kafel_name; /* name for kafel policy ("newstat", "sendmsg") */
const char* display_name; /* name for stats output ("stat", "sendmsg") */
SyscallCategory category;
ArgRole args[6]; /* role of each syscall argument */
};
/* Shorthand for table readability */
using A = ArgRole;
/*
* THE SINGLE SOURCE OF TRUTH for which syscalls are traced via seccomp unotify.
*
* To add a new traced syscall: add one entry here.
* The kafel BPF policy, name lookup, and argument decoding all derive from this table.
*/
static constexpr SyscallDef kTracedSyscalls[] = {
/* FS - arg0 = path (AT_FDCWD implied) */
{__NR_open, "open", "open", SyscallCategory::FS,
{A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_stat, "newstat", "stat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_lstat, "newlstat", "lstat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_access, "access", "access", SyscallCategory::FS,
{A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chmod, "chmod", "chmod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chown, "chown", "chown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
{__NR_lchown, "lchown", "lchown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
{__NR_readlink, "readlink", "readlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mkdir, "mkdir", "mkdir", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mknod, "mknod", "mknod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_rmdir, "rmdir", "rmdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_unlink, "unlink", "unlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_execve, "execve", "execve", SyscallCategory::FS,
{A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chdir, "chdir", "chdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chroot, "chroot", "chroot", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
/* FS - arg0 = dirfd, arg1 = path */
{__NR_openat, "openat", "openat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP}},
{__NR_openat2, "openat2", "openat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OHOW, A::SKIP, A::SKIP, A::SKIP}},
{__NR_newfstatat, "newfstatat", "newfstatat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_faccessat, "faccessat", "faccessat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP}},
{__NR_fchmodat, "fchmodat", "fchmodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_fchownat, "fchownat", "fchownat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::UID, A::GID, A::SKIP, A::SKIP}},
{__NR_readlinkat, "readlinkat", "readlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mkdirat, "mkdirat", "mkdirat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mknodat, "mknodat", "mknodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_unlinkat, "unlinkat", "unlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_execveat, "execveat", "execveat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP}},
/* FS - two paths */
{__NR_rename, "rename", "rename", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_link, "link", "link", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_symlink, "symlink", "symlink", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
/* FS - two dirfd+path pairs */
{__NR_renameat, "renameat", "renameat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
{__NR_renameat2, "renameat2", "renameat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
{__NR_linkat, "linkat", "linkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
/* symlinkat: arg0=target(CWD), arg1=newdirfd, arg2=newpath */
{__NR_symlinkat, "symlinkat", "symlinkat", SyscallCategory::FS,
{A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP}},
/* NET - sockaddr-based */
{__NR_connect, "connect", "connect", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
{__NR_bind, "bind", "bind", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
{__NR_sendto, "sendto", "sendto", SyscallCategory::NET,
{A::FD, A::SKIP, A::SKIP, A::IFLAGS, A::SADDR, A::ALEN}},
/* NET - socket creation */
{__NR_socket, "socket", "socket", SyscallCategory::NET,
{A::DOMAIN, A::STYPE, A::PROTO, A::SKIP, A::SKIP, A::SKIP}},
};
constexpr size_t kTracedSyscallCount = sizeof(kTracedSyscalls) / sizeof(kTracedSyscalls[0]);
/*
* Build the kafel policy string from the table.
* Called by sandbox.cc - no manual syscall list maintenance needed.
*/
inline std::string buildKafelPolicy() {
std::string p = "POLICY unotify {\n USER_NOTIF {\n";
for (size_t i = 0; i < kTracedSyscallCount; i++) {
if (i > 0) p += ", ";
p += kTracedSyscalls[i].kafel_name;
}
p += "\n }\n}\nUSE unotify DEFAULT ALLOW\n";
return p;
}
} // namespace unotify
#endif /* NSJAIL_UNOTIFY_SYSCALL_DEFS_H */

126
unotify/unotify.cc Normal file
View File

@@ -0,0 +1,126 @@
/*
* Seccomp Unotify subsystem for nsjail.
* This module allows nsjail to observe and log syscalls made by the sandboxed
* process using the SECCOMP_USER_NOTIF feature. It runs a background thread
* that reads notifications, decodes arguments, and aggregates statistics.
*/
#include "unotify.h"
#include <linux/seccomp.h>
#include <poll.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <unistd.h>
#include <thread>
#include "logs.h"
#include "unotify/record.h"
#include "unotify/stats.h"
#include "unotify/syscall.h"
#include "util.h"
namespace unotify {
#ifndef SECCOMP_IOCTL_NOTIF_RECV
#define SECCOMP_IOCTL_NOTIF_RECV SECCOMP_IOWR(0, struct seccomp_notif)
#define SECCOMP_IOCTL_NOTIF_SEND SECCOMP_IOWR(1, struct seccomp_notif_resp)
#define SECCOMP_IOCTL_NOTIF_ID_VALID SECCOMP_IOWR(2, __u64)
#endif
#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
#define SECCOMP_USER_NOTIF_FLAG_CONTINUE (1UL << 0)
#endif
static int unotif_fd = -1;
static std::thread* worker_thread = nullptr;
static bool isTargetAlive(int fd, __u64 last_id) {
return ioctl(fd, SECCOMP_IOCTL_NOTIF_ID_VALID, &last_id) == 0;
}
static void threadMain() {
LOG_I("Started unotify loop");
/*
* When last_id is 0 (no notification received yet), NOTIF_ID_VALID
* will always fail since ID 0 is never valid. This correctly exits
* the loop when NOTIF_RECV has never succeeded (child died before
* making any traced syscall).
*/
__u64 last_id = 0;
while (true) {
struct pollfd pfd = {.fd = unotif_fd, .events = POLLIN, .revents = 0};
int ret = poll(&pfd, 1, -1);
if (ret == -1) {
if (errno == EINTR) continue;
PLOG_E("poll failed");
if (!isTargetAlive(unotif_fd, last_id)) break;
continue;
}
struct seccomp_notif req = {};
if (ioctl(unotif_fd, SECCOMP_IOCTL_NOTIF_RECV, &req) == -1) {
if (errno == EINTR) continue;
PLOG_D("SECCOMP_IOCTL_NOTIF_RECV");
if (!isTargetAlive(unotif_fd, last_id)) break;
continue;
}
last_id = req.id;
LOG_D("Received seccomp notification for syscall %d", req.data.nr);
SyscallRecord rec;
parseSyscall(&req, &rec);
addStat(rec);
if (!isTargetAlive(unotif_fd, req.id)) {
break;
}
struct seccomp_notif_resp resp = {};
resp.id = req.id;
resp.flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE;
if (ioctl(unotif_fd, SECCOMP_IOCTL_NOTIF_SEND, &resp) == -1) {
if (errno != ENOENT) {
PLOG_E("SECCOMP_IOCTL_NOTIF_SEND failed");
}
if (!isTargetAlive(unotif_fd, req.id)) break;
}
}
}
bool start(nsj_t* nsj, int fd) {
if (worker_thread) {
LOG_W("unotify::start() called while already running. "
"Concurrent tracing in LISTEN mode is not yet supported. "
"Closing notification fd for this process.");
close(fd);
return true;
}
unotif_fd = fd;
worker_thread = new std::thread(threadMain);
return true;
}
/*
* Called after killAndReapAll(). The worker thread exits its loop when
* isTargetAlive() (SECCOMP_IOCTL_NOTIF_ID_VALID) reports the target is dead.
*/
void stop(nsj_t* nsj) {
if (worker_thread) {
worker_thread->join();
delete worker_thread;
worker_thread = nullptr;
}
if (unotif_fd != -1) {
close(unotif_fd);
unotif_fd = -1;
}
printStats(nsj);
}
} // namespace unotify

14
unotify/unotify.h Normal file
View File

@@ -0,0 +1,14 @@
#ifndef NSJAIL_UNOTIFY_H
#define NSJAIL_UNOTIFY_H
#include "nsjail.h"
namespace unotify {
bool start(nsj_t* nsj, int unotif_fd);
void stop(nsj_t* nsj);
/* Note: printStats is in unotify/stats.h, exposed to nsjail.cc */
} // namespace unotify
#endif /* NSJAIL_UNOTIFY_H */

72
unotify/unotify.proto Normal file
View File

@@ -0,0 +1,72 @@
syntax = "proto2";
package unotify;
message Stat {
message Syscall {
optional string name = 1;
// Human-readable stringified arguments (e.g. parsed flags or mode changes)
repeated string args = 2;
optional uint64 count = 6 [default = 0];
}
message Path {
enum Type {
UNKNOWN = 0;
NONEXISTENT = 1;
REGULAR = 2;
DIR = 3;
CHR = 4;
BLK = 5;
FIFO = 6;
LINK = 7;
SOCK = 8;
}
// Resolved absolute path
optional string path = 1;
// Whether this path maps to a valid inode inside the jailed mount namespace
optional bool exists_in_jail = 7 [default = false];
// Whether this path maps to a valid inode in the host's initial mount namespace
optional bool exists_in_main = 8 [default = false];
optional Type jail_type = 2 [default = UNKNOWN];
optional Type main_type = 3 [default = UNKNOWN];
enum Mode {
UNSPECIFIED = 0;
RDONLY = 1;
WRONLY = 2;
RDWR = 3;
}
// Aggregated access intent based on syscall flags (e.g., O_RDONLY vs O_RDWR)
optional Mode mode = 4 [default = UNSPECIFIED];
optional uint64 count = 5 [default = 0];
repeated Syscall syscall = 6;
}
message NetResource {
enum Type {
UNKNOWN = 0;
IPV4 = 1;
IPV6 = 2;
UNIX = 3;
NETLINK = 4;
}
// Network protocol domain
optional Type type = 1 [default = UNKNOWN];
// Stringified IP address or generic identifier
optional string endpoint = 2;
optional uint32 port = 6;
// Associated filesystem path for AF_UNIX sockets
optional Path socket_path = 3;
optional uint64 count = 4 [default = 0];
repeated Syscall syscall = 5;
}
repeated Path fs_access = 1;
repeated NetResource net_access = 2;
}
// vim: set noexpandtab ts=4 sw=4: