Github/nsjail
1
0
Fork 0
mirror of https://github.com/google/nsjail.git synced 2026-08-30 18:41:30 -07:00
Code Issues Packages Projects Releases Wiki Activity

nsjail 4.0: monitor thread modernization, nstun/sockproxy/unotify refactoring

Browse Source 
Major changes:
- Co-developed with gemini-cli (reviewed by a human)
- Per-child epoll-based monitor thread with thread_local context
- Replaced fixed-size arrays with STL containers (unordered_map, vector)
- Zero-copy splice proxy for listen mode (sockproxy)
- Seccomp unotify integrated into epoll loop (no dedicated thread)
- nstun: SOCKS5/HTTP CONNECT proxy, MemPool allocator, IPv6 support
- close_range() for FD cleanup
- Test coverage for nstun standalone/listen modes
This commit is contained in:
Robert Swiecki
2026-04-10 20:00:34 +02:00
parent 91010b3cdf
commit 273a691dd5
55 changed files with 6068 additions and 3700 deletions
Download Patch File Download Diff File Expand all files Collapse all files

139
Makefile
View File

@@ -30,10 +30,19 @@ COMMON_FLAGS += -O2 -c \
-Ikafel/include
CXXFLAGS += $(USER_DEFINES) $(COMMON_FLAGS) $(PROTOBUF_CFLAGS) -I. \
-std=c++20 -fno-exceptions -Wno-unused -Wno-unused-parameter -Wno-c99-designator
-std=c++20 -fno-exceptions -Wno-unused -Wno-unused-parameter
ifneq ($(findstring clang,$(CXX)),)
CXXFLAGS += -Wno-c99-designator
endif
LDFLAGS += -pie -Wl,-z,noexecstack -lpthread $(PROTOBUF_LIBS)
ifdef USE_ASAN
COMMON_FLAGS += -fsanitize=address
LDFLAGS += -fsanitize=address
endif
ifeq ($(NL3_EXISTS), yes)
CXXFLAGS += $(shell pkg-config --cflags libnl-route-3.0)
LDFLAGS += $(shell pkg-config --libs libnl-route-3.0)
@@ -61,7 +70,7 @@ ifneq ($(PASTA_BIN_PATH),)
CXXFLAGS += -DPASTA_BIN_PATH='"$(PASTA_BIN_PATH)"'
endif
SRCS_CXX = caps.cc cgroup.cc cgroup2.cc cmdline.cc config.cc contain.cc cpu.cc logs.cc mnt.cc mnt_legacy.cc mnt_newapi.cc net.cc nsjail.cc pid.cc sandbox.cc subproc.cc uts.cc user.cc unotify/unotify.cc unotify/stats.cc unotify/syscall.cc util.cc nstun/nstun.cc nstun/policy.cc nstun/encap.cc nstun/iface.cc nstun/tun.cc nstun/ip.cc nstun/icmp.cc nstun/udp.cc nstun/tcp.cc
SRCS_CXX = monitor.cc sockproxy/sockproxy.cc caps.cc cgroup.cc cgroup2.cc cmdline.cc config.cc contain.cc cpu.cc logs.cc mnt.cc mnt_legacy.cc mnt_newapi.cc net.cc nsjail.cc pid.cc sandbox.cc subproc.cc uts.cc user.cc unotify/unotify.cc unotify/stats.cc unotify/syscall.cc util.cc nstun/nstun.cc nstun/policy.cc nstun/encap.cc nstun/iface.cc nstun/tun.cc nstun/ip.cc nstun/icmp.cc nstun/udp.cc nstun/tcp.cc
SRCS_PROTO = config.proto unotify/unotify.proto
SRCS_PB_CXX = $(SRCS_PROTO:.proto=.pb.cc)
@@ -185,8 +194,12 @@ test: $(BIN)
$(call run_test, ./nsjail --config tests/connect.cfg -q -t 3 -- /bin/bash -c 'host -T dns.google 8.8.8.8 && exit 77', 77)
$(call run_test, ./nsjail --config tests/connect.cfg -q -t 3 -- /bin/bash -c 'host -T dns.google 2001:4860:4860::8888 && exit 77', 77)
# --- Nstun standalone / proxy mode tests ---
$(call run_test, ./nsjail --config tests/nstun.cfg -Mo -q -t 2 --seccomp_unotify -- /bin/bash -c 'exit 77', 77)
$(call run_test, { ./nsjail --config tests/nstun.cfg -Ml --port 31338 -q -t 5 --seccomp_unotify -- /bin/bash -c "sleep 10" & }; sleep 2; echo -n "GET / HTTP/1.0\r\n\r\n" | nc 127.0.0.1 31338 >/dev/null 2>&1 && exit 77, 77)
# --- HOST_TO_GUEST TCP inbound proxy test (IPv4 + IPv6) ---
$(call run_test, { ./nsjail --config tests/dns_http_host_to_guest.cfg -q -t 3 & }; sleep 1; wget -4 -q -O /dev/null --timeout=5 http://127.0.0.1:8080/ && wget -6 -q -O /dev/null --timeout=5 http://[::1]:8080/ && exit 77, 77)
$(call run_test, { ./nsjail --config tests/dns_http_host_to_guest.cfg -q -t 5 & }; sleep 2; wget -4 -q -O /dev/null --timeout=5 http://127.0.0.1:8080/ && wget -6 -q -O /dev/null --timeout=5 http://[::1]:8080/ && exit 77, 77)
# --- --experimental_mnt=old ---
$(call run_test, ./nsjail $(OLD_EF) -q -Mo --rw --chroot / --user 99999 --group 99999 -- /bin/bash -c 'touch $(HOME)/nsjail_test && exit 77', 77)
@@ -202,12 +215,12 @@ test: $(BIN)
$(call run_test, rm -f /run/user/$(UID)/nsjail_test2, 0)
$(call run_test, ./nsjail $(OLD_EF) --config configs/bash-with-fake-geteuid.cfg -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(OLD_EF) --config configs/bash-with-fake-geteuid.json -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(OLD_EF) --config configs/static-busybox-with-execveat.cfg -q -t 1, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/static-busybox-with-execveat.cfg -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(OLD_EF) --config configs/home-documents-with-xorg-no-net.cfg -q -- /bin/true, 0)
$(call run_test, ./nsjail $(OLD_EF) --config configs/home-documents-with-xorg-no-net.cfg -q -- /bin/false, 1)
$(call run_test, ./nsjail $(OLD_EF) --config configs/firefox-with-net-X11.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/firefox-with-net-wayland.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/chromium-with-net-wayland.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/firefox-with-net-X11.cfg -q -t 3, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/firefox-with-net-wayland.cfg -q -t 3, 137)
$(call run_test, ./nsjail $(OLD_EF) --config configs/chromium-with-net-wayland.cfg -q -t 5, 137)
# --- --experimental_mnt=new ---
$(call run_test, ./nsjail $(NEW_EF) -q -Mo --rw --chroot / --user 99999 --group 99999 -- /bin/bash -c 'touch $(HOME)/nsjail_test && exit 77', 77)
@@ -223,12 +236,12 @@ test: $(BIN)
$(call run_test, rm -f /run/user/$(UID)/nsjail_test2, 0)
$(call run_test, ./nsjail $(NEW_EF) --config configs/bash-with-fake-geteuid.cfg -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(NEW_EF) --config configs/bash-with-fake-geteuid.json -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(NEW_EF) --config configs/static-busybox-with-execveat.cfg -q -t 1, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/static-busybox-with-execveat.cfg -q -t 1 < /dev/null, 0)
$(call run_test, ./nsjail $(NEW_EF) --config configs/home-documents-with-xorg-no-net.cfg -q -- /bin/true, 0)
$(call run_test, ./nsjail $(NEW_EF) --config configs/home-documents-with-xorg-no-net.cfg -q -- /bin/false, 1)
$(call run_test, ./nsjail $(NEW_EF) --config configs/firefox-with-net-X11.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/firefox-with-net-wayland.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/chromium-with-net-wayland.cfg -q -t 2, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/firefox-with-net-X11.cfg -q -t 3, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/firefox-with-net-wayland.cfg -q -t 3, 137)
$(call run_test, ./nsjail $(NEW_EF) --config configs/chromium-with-net-wayland.cfg -q -t 5, 137)
@echo ""
@echo "========================================"
@@ -239,61 +252,75 @@ test: $(BIN)
# Dependencies (Generated by makedepend)
# DO NOT DELETE THIS LINE -- make depend depends on it.
caps.o: caps.h nsjail.h config.pb.h logs.h macros.h util.h
cgroup.o: cgroup.h nsjail.h config.pb.h logs.h util.h
cgroup2.o: cgroup2.h nsjail.h config.pb.h logs.h util.h
monitor.o: monitor.h logs.h macros.h missing_defs.h net.h nsjail.h
monitor.o: config.pb.h nstun/nstun.h sockproxy/sockproxy.h subproc.h
monitor.o: unotify/unotify.h util.h
sockproxy/sockproxy.o: sockproxy/sockproxy.h logs.h monitor.h util.h
sockproxy/sockproxy.o: missing_defs.h nsjail.h config.pb.h
caps.o: caps.h nsjail.h config.pb.h logs.h macros.h missing_defs.h util.h
cgroup.o: cgroup.h nsjail.h config.pb.h logs.h util.h missing_defs.h
cgroup2.o: cgroup2.h nsjail.h config.pb.h logs.h util.h missing_defs.h
cmdline.o: cmdline.h nsjail.h config.pb.h caps.h config.h logs.h macros.h
cmdline.o: mnt.h mnt_newapi.h user.h util.h
cmdline.o: missing_defs.h mnt.h mnt_newapi.h user.h util.h
config.o: config.h nsjail.h config.pb.h caps.h cmdline.h logs.h macros.h
config.o: mnt.h user.h util.h
config.o: missing_defs.h mnt.h user.h util.h
contain.o: contain.h nsjail.h config.pb.h caps.h cgroup.h cgroup2.h config.h
contain.o: cpu.h logs.h macros.h mnt.h net.h pid.h user.h util.h uts.h
cpu.o: cpu.h nsjail.h config.pb.h logs.h util.h
logs.o: logs.h macros.h util.h nsjail.h config.pb.h
mnt.o: mnt.h nsjail.h config.pb.h logs.h macros.h mnt_legacy.h mnt_newapi.h
mnt.o: subproc.h util.h
mnt_legacy.o: mnt_legacy.h mnt.h nsjail.h config.pb.h logs.h macros.h util.h
mnt_newapi.o: mnt_newapi.h mnt.h nsjail.h config.pb.h logs.h util.h
net.o: net.h nsjail.h config.pb.h logs.h macros.h nstun/nstun.h util.h
nsjail.o: nsjail.h config.pb.h cgroup2.h cmdline.h logs.h macros.h net.h
nsjail.o: sandbox.h subproc.h unotify/unotify.h util.h
pid.o: pid.h nsjail.h config.pb.h logs.h subproc.h
sandbox.o: sandbox.h nsjail.h config.pb.h subproc.h kafel/include/kafel.h
sandbox.o: logs.h unotify/syscall_defs.h util.h
subproc.o: subproc.h nsjail.h config.pb.h cgroup.h cgroup2.h contain.h logs.h
subproc.o: macros.h net.h nstun/nstun.h sandbox.h unotify/unotify.h user.h
subproc.o: util.h
contain.o: cpu.h logs.h macros.h missing_defs.h mnt.h net.h monitor.h pid.h
contain.o: user.h util.h uts.h
cpu.o: cpu.h nsjail.h config.pb.h logs.h util.h missing_defs.h
logs.o: logs.h macros.h util.h missing_defs.h nsjail.h config.pb.h
mnt.o: mnt.h missing_defs.h nsjail.h config.pb.h logs.h macros.h mnt_legacy.h
mnt.o: mnt_newapi.h subproc.h monitor.h util.h
mnt_legacy.o: mnt_legacy.h mnt.h missing_defs.h nsjail.h config.pb.h logs.h
mnt_legacy.o: macros.h util.h
mnt_newapi.o: mnt_newapi.h mnt.h missing_defs.h nsjail.h config.pb.h logs.h
mnt_newapi.o: macros.h util.h
net.o: net.h monitor.h nsjail.h config.pb.h logs.h macros.h missing_defs.h
net.o: nstun/nstun.h subproc.h util.h
nsjail.o: nsjail.h config.pb.h cgroup2.h cmdline.h logs.h macros.h
nsjail.o: missing_defs.h monitor.h net.h sandbox.h subproc.h unotify/stats.h
nsjail.o: unotify/record.h unotify/unotify.pb.h util.h
pid.o: pid.h nsjail.h config.pb.h logs.h subproc.h monitor.h
sandbox.o: sandbox.h nsjail.h config.pb.h subproc.h monitor.h
sandbox.o: kafel/include/kafel.h logs.h missing_defs.h unotify/syscall_defs.h
sandbox.o: missing_defs.h util.h
subproc.o: subproc.h monitor.h nsjail.h config.pb.h cgroup.h cgroup2.h
subproc.o: contain.h logs.h macros.h missing_defs.h net.h nstun/nstun.h
subproc.o: sandbox.h user.h util.h
uts.o: uts.h nsjail.h config.pb.h logs.h
user.o: user.h nsjail.h config.pb.h logs.h macros.h subproc.h util.h
user.o: user.h nsjail.h config.pb.h logs.h macros.h subproc.h monitor.h
user.o: util.h missing_defs.h
unotify/unotify.o: unotify/unotify.h nsjail.h config.pb.h logs.h
unotify/unotify.o: unotify/record.h unotify/unotify.pb.h unotify/stats.h
unotify/unotify.o: unotify/syscall.h util.h
unotify/unotify.o: missing_defs.h monitor.h unotify/record.h
unotify/unotify.o: unotify/unotify.pb.h unotify/stats.h unotify/syscall.h
unotify/unotify.o: util.h
unotify/stats.o: unotify/stats.h nsjail.h config.pb.h unotify/record.h
unotify/stats.o: unotify/unotify.pb.h logs.h util.h
unotify/stats.o: unotify/unotify.pb.h logs.h util.h missing_defs.h
unotify/syscall.o: unotify/syscall.h unotify/record.h unotify/unotify.pb.h
unotify/syscall.o: logs.h macros.h unotify/syscall_defs.h util.h nsjail.h
unotify/syscall.o: config.pb.h
util.o: util.h nsjail.h config.pb.h logs.h macros.h
nstun/nstun.o: nstun/nstun.h nstun/core.h nstun/net_defs.h nstun/icmp.h
nstun/nstun.o: nstun/iface.h nstun/ip.h logs.h macros.h nstun/policy.h
nstun/nstun.o: nstun/tcp.h nstun/tun.h nstun/udp.h util.h nsjail.h
nstun/nstun.o: config.pb.h
nstun/policy.o: nstun/policy.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/policy.o: logs.h config.pb.h nsjail.h
nstun/encap.o: nstun/encap.h nstun/net_defs.h logs.h
unotify/syscall.o: logs.h macros.h missing_defs.h unotify/syscall_defs.h
unotify/syscall.o: missing_defs.h util.h nsjail.h config.pb.h
util.o: util.h missing_defs.h nsjail.h config.pb.h logs.h macros.h
nstun/nstun.o: nstun/nstun.h monitor.h nstun/core.h nstun/net_defs.h
nstun/nstun.o: nstun/encap.h nstun/icmp.h nstun/iface.h nstun/ip.h logs.h
nstun/nstun.o: macros.h nsjail.h config.pb.h nstun/policy.h nstun/tcp.h
nstun/nstun.o: nstun/tun.h nstun/udp.h util.h missing_defs.h
nstun/policy.o: nstun/policy.h config.pb.h nstun/core.h nstun/net_defs.h
nstun/policy.o: nstun/nstun.h monitor.h nstun/encap.h logs.h nsjail.h
nstun/encap.o: nstun/encap.h nstun/net_defs.h logs.h macros.h
nstun/iface.o: nstun/iface.h logs.h macros.h nstun/net_defs.h nsjail.h
nstun/iface.o: config.pb.h nstun/nstun.h
nstun/iface.o: config.pb.h nstun/nstun.h monitor.h
nstun/tun.o: nstun/tun.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/tun.o: nstun/icmp.h nstun/ip.h logs.h
nstun/ip.o: nstun/ip.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/ip.o: nstun/icmp.h logs.h nstun/tcp.h nstun/udp.h
nstun/icmp.o: nstun/icmp.h nstun/core.h nstun/net_defs.h nstun/nstun.h logs.h
nstun/icmp.o: macros.h nstun/policy.h nstun/tun.h
nstun/tun.o: monitor.h nstun/encap.h nstun/icmp.h nstun/ip.h logs.h
nstun/ip.o: nstun/ip.h nstun/core.h nstun/net_defs.h nstun/nstun.h monitor.h
nstun/ip.o: nstun/encap.h nstun/icmp.h logs.h nstun/tcp.h nstun/udp.h
nstun/icmp.o: nstun/icmp.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/icmp.o: monitor.h nstun/encap.h logs.h macros.h nstun/policy.h
nstun/icmp.o: config.pb.h nstun/tun.h
nstun/udp.o: nstun/udp.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/udp.o: nstun/encap.h nstun/icmp.h logs.h macros.h nstun/policy.h
nstun/udp.o: nstun/tun.h
nstun/udp.o: monitor.h nstun/encap.h nstun/icmp.h logs.h macros.h
nstun/udp.o: nstun/policy.h config.pb.h nstun/tun.h
nstun/tcp.o: nstun/tcp.h nstun/core.h nstun/net_defs.h nstun/nstun.h
nstun/tcp.o: nstun/encap.h logs.h macros.h nstun/policy.h nstun/tun.h util.h
nstun/tcp.o: nsjail.h config.pb.h
nstun/tcp.o: monitor.h nstun/encap.h logs.h macros.h nstun/policy.h
nstun/tcp.o: config.pb.h nstun/tun.h util.h missing_defs.h nsjail.h
config.pb.o: config.pb.h
unotify/unotify.pb.o: unotify/unotify.pb.h

19
caps.cc
View File

@@ -33,21 +33,9 @@
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "util.h"
#if !defined(CAP_AUDIT_READ)
#define CAP_AUDIT_READ 37
#endif /* !defined(CAP_AUDIT_READ) */
#if !defined(CAP_PERFMON)
#define CAP_PERFMON 38
#endif /* !defined(CAP_PERFMON) */
#if !defined(CAP_BPF)
#define CAP_BPF 39
#endif /* !defined(CAP_BPF) */
#if !defined(CAP_CHECKPOINT_RESTORE)
#define CAP_CHECKPOINT_RESTORE 40
#endif /* !defined(CAP_CHECKPOINT_RESTORE) */
namespace caps {
struct {
@@ -174,11 +162,6 @@ static void setInheritable(cap_user_data_t cap_data, unsigned int cap) {
cap_data[off_byte].inheritable |= mask;
}
#if !defined(PR_CAP_AMBIENT)
#define PR_CAP_AMBIENT 47
#define PR_CAP_AMBIENT_RAISE 2
#define PR_CAP_AMBIENT_CLEAR_ALL 4
#endif /* !defined(PR_CAP_AMBIENT) */
static bool initNsKeepCaps(cap_user_data_t cap_data) {
/* Copy all permitted caps to the inheritable set */
std::string dbgmsg1;

16
cmdline.cc
View File

@@ -52,6 +52,7 @@
#include "config.h"
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt.h"
#include "mnt_newapi.h"
#include "user.h"
@@ -405,13 +406,6 @@ static bool setupArgv(nsj_t* nsj, int argc, char** argv, int optind) {
}
if (nsj->njc.exec_bin().exec_fd()) {
#if !defined(__NR_execveat)
LOG_E("Your nsjail is compiled without support for the execveat() "
"syscall, "
"yet you "
"specified the --execute_fd flag");
return false;
#endif /* !defined(__NR_execveat) */
if ((nsj->exec_fd = TEMP_FAILURE_RETRY(open(nsj->njc.exec_bin().path().c_str(),
O_RDONLY | O_PATH | O_CLOEXEC))) == -1) {
PLOG_W("Couldn't open %s file", QC(nsj->njc.exec_bin().path()));
@@ -493,18 +487,18 @@ std::unique_ptr<nsj_t> parseArgs(int argc, char* argv[]) {
/* Generate options array for getopt_long. */
const size_t options_length = ARR_SZ(custom_opts) + 1;
struct option opts[options_length];
std::vector<struct option> opts(options_length);
for (unsigned i = 0; i < ARR_SZ(custom_opts); i++) {
opts[i] = custom_opts[i].opt;
}
/* Lastly, NULL option as a terminator */
struct option terminator = {NULL, 0, NULL, 0};
memcpy(&opts[options_length - 1].name, &terminator, sizeof(terminator));
opts[options_length - 1] = {nullptr, 0, nullptr, 0};
int opt_index = 0;
for (;;) {
int c = getopt_long(argc, argv,
"x:H:D:C:c:p:i:u:g:l:L:t:M:NdvqQeh?E:R:B:T:m:s:P:I:U:G:", opts, &opt_index);
"x:H:D:C:c:p:i:u:g:l:L:t:M:NdvqQeh?E:R:B:T:m:s:P:I:U:G:", opts.data(),
&opt_index);
if (c == -1) {
break;
}

3
config.cc
View File

@@ -39,6 +39,7 @@
#include "config.pb.h"
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt.h"
#include "user.h"
#include "util.h"
@@ -156,7 +157,7 @@ static void logHandler(
#endif /* NSJAIL_HAS_PROTOBUF_LOG_HANDLER */
static void flushLog() {
for (auto message : error_messages) {
for (const auto& message : error_messages) {
LOG_W("ProtoTextFormat: %s", message.c_str());
}
error_messages.clear();

1
configs/chromium-with-net-wayland.cfg
View File

@@ -187,4 +187,5 @@ exec_bin {
path: "/usr/lib/chromium/chromium"
arg: "--ozone-platform=wayland"
arg: "--incognito"
arg: "https://fast.com"
}

1
configs/firefox-with-net-X11.cfg
View File

@@ -190,4 +190,5 @@ seccomp_string: "DEFAULT ALLOW"
exec_bin {
path: "/usr/lib/firefox/firefox"
arg: "-private"
arg: "https://fast.com"
}

1
configs/firefox-with-net-wayland.cfg
View File

@@ -185,4 +185,5 @@ seccomp_string: "DEFAULT ALLOW"
exec_bin {
path: "/usr/lib/firefox/firefox"
arg: "-private"
arg: "https://fast.com"
}

206
contain.cc
View File

@@ -26,6 +26,9 @@
#include <fcntl.h>
#include <inttypes.h>
#include <limits.h>
#if __has_include(<linux/close_range.h>)
#include <linux/close_range.h>
#endif
#include <signal.h>
#include <stdbool.h>
#include <stddef.h>
@@ -46,6 +49,7 @@
#include "cpu.h"
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt.h"
#include "net.h"
#include "pid.h"
@@ -76,9 +80,6 @@ static bool containInitCgroupNs(void) {
}
static bool containDropPrivs(nsj_t* nsj) {
#ifndef PR_SET_NO_NEW_PRIVS
#define PR_SET_NO_NEW_PRIVS 38
#endif
if (!nsj->njc.disable_no_new_privs()) {
if (prctl(PR_SET_NO_NEW_PRIVS, 1UL, 0UL, 0UL, 0UL) == -1) {
/* Only new kernels support it */
@@ -98,24 +99,24 @@ static bool containPrepareEnv(nsj_t* nsj) {
PLOG_E("prctl(PR_SET_PDEATHSIG, SIGKILL)");
return false;
}
unsigned long personality = 0;
unsigned long pers = 0;
if (nsj->njc.persona_addr_compat_layout()) {
personality |= ADDR_COMPAT_LAYOUT;
pers |= ADDR_COMPAT_LAYOUT;
}
if (nsj->njc.persona_mmap_page_zero()) {
personality |= MMAP_PAGE_ZERO;
pers |= MMAP_PAGE_ZERO;
}
if (nsj->njc.persona_read_implies_exec()) {
personality |= READ_IMPLIES_EXEC;
pers |= READ_IMPLIES_EXEC;
}
if (nsj->njc.persona_addr_limit_3gb()) {
personality |= ADDR_LIMIT_3GB;
pers |= ADDR_LIMIT_3GB;
}
if (nsj->njc.persona_addr_no_randomize()) {
personality |= ADDR_NO_RANDOMIZE;
pers |= ADDR_NO_RANDOMIZE;
}
if (personality && ::personality(personality) == -1) {
PLOG_E("personality(%lx)", personality);
if (pers && personality(pers) == -1) {
PLOG_E("personality(%lx)", pers);
return false;
}
LOG_D("setpriority(%d)", nsj->njc.nice_level());
@@ -126,6 +127,7 @@ static bool containPrepareEnv(nsj_t* nsj) {
if (!nsj->njc.skip_setsid()) {
setsid();
}
return true;
}
@@ -221,10 +223,10 @@ static bool containSetLimits(nsj_t* nsj) {
return true;
}
static bool containPassFd(nsj_t* nsj, int fd) {
return (std::find(nsj->openfds.begin(), nsj->openfds.end(), fd) != nsj->openfds.end());
}
/*
* Marks FDs for close-on-exec, or clears it for FDs that should be passed.
* exec(2) handles the actual closing.
*/
static bool containMakeFdCOE(int fd, bool pass_fd) {
int flags = TEMP_FAILURE_RETRY(fcntl(fd, F_GETFD, 0));
if (flags == -1) {
@@ -252,100 +254,104 @@ static bool containMakeFdCOE(int fd, bool pass_fd) {
return true;
}
static bool containMakeFdsCOECloseRange(nsj_t* nsj) {
RETURN_ON_FAILURE(util::makeRangeCOE(0U, ~0U));
for (const auto fd : nsj->openfds) {
RETURN_ON_FAILURE(containMakeFdCOE(fd, /* pass_fd= */ true));
}
return true;
}
static bool containMakeFdsCOENaive(nsj_t* nsj) {
/*
* Don't use getrlimit(RLIMIT_NOFILE) here, as it can return an artifically small value
* (e.g. 32), which could be smaller than a maximum assigned number to file-descriptors
* in this process. Just use some reasonably sane value (e.g. 1024)
*/
for (unsigned fd = 0; fd < 1024; fd++) {
RETURN_ON_FAILURE(containMakeFdCOE(fd, containPassFd(nsj, fd)));
}
return true;
}
static bool containMakeFdsCOEProc(nsj_t* nsj) {
int dirfd = open("/proc/self/fd", O_DIRECTORY | O_RDONLY | O_CLOEXEC);
if (dirfd == -1) {
PLOG_D("open('/proc/self/fd', O_DIRECTORY|O_RDONLY|O_CLOEXEC)");
return false;
}
DIR* dir = fdopendir(dirfd);
if (dir == nullptr) {
PLOG_W("fdopendir(fd=%d)", dirfd);
close(dirfd);
return false;
}
/* Make all fds above stderr close-on-exec */
for (;;) {
errno = 0;
struct dirent* entry = readdir(dir);
if (entry == nullptr && errno != 0) {
PLOG_D("readdir('/proc/self/fd')");
closedir(dir);
return false;
}
if (entry == nullptr) {
break;
}
if (util::StrEq(".", entry->d_name)) {
continue;
}
if (util::StrEq("..", entry->d_name)) {
continue;
}
errno = 0;
int fd = strtoimax(entry->d_name, NULL, 10);
if (errno != 0) {
PLOG_W("Cannot convert /proc/self/fd/%s to a number", entry->d_name);
continue;
}
int flags = TEMP_FAILURE_RETRY(fcntl(fd, F_GETFD, 0));
if (flags == -1) {
PLOG_D("fcntl(fd=%d, F_GETFD, 0)", fd);
closedir(dir);
return false;
}
RETURN_ON_FAILURE(containMakeFdCOE(fd, containPassFd(nsj, fd)));
}
closedir(dir);
return true;
}
/*
* Sets the Close-On-Exec (COE) flag on all file descriptors except those explicitly
* marked to be passed to the sandbox.
*
* To optimize performance, we utilize the close_range() syscall. Because we are
* single-threaded in the child process just before execve(), we can be _cheeky_ -
* we blanket-apply CLOSE_RANGE_CLOEXEC to the entire descriptor table (0 to ~0U),
* and then simply clear the flag on the exact FDs we wish to pass.
*/
static bool containMakeFdsCOE(nsj_t* nsj) {
if (containMakeFdsCOECloseRange(nsj)) {
return true;
if (util::syscall(__NR_close_range, 0, ~0U, CLOSE_RANGE_CLOEXEC) == -1) {
PLOG_E("close_range(0, ~0U, CLOSE_RANGE_CLOEXEC)");
return false;
}
if (containMakeFdsCOEProc(nsj)) {
return true;
for (const auto fd : nsj->openfds) {
if (fd >= 0) {
containMakeFdCOE(fd, /* pass_fd= */ true);
}
}
if (containMakeFdsCOENaive(nsj)) {
return true;
}
LOG_E("Couldn't mark relevant file-descriptors as close-on-exec with any known method");
return false;
return true;
}
bool setupFD(nsj_t* nsj, int fd_in, int fd_out, int fd_err) {
/*
* (Violently) closes all file descriptors that are not explicitly required to survive
* the containment boundary.
*
* Unlike COE above, closing an FD is a destructive and irreversible operation.
* We cannot "close everything and revert". We must build a sorted list of critical
* FDs (standard I/O, IPC sockets, logging, etc.) and safely jump over them by
* calling close_range() on the numerical "gaps" between our preserved FDs.
*/
static bool containCloseFDs(nsj_t* nsj, int ipc_fd) {
std::vector<unsigned int> keep_fds;
/* Core standard I/O */
keep_fds.push_back(STDIN_FILENO);
keep_fds.push_back(STDOUT_FILENO);
keep_fds.push_back(STDERR_FILENO);
/* Crucial infrastructure */
if (logs::logFd() > STDERR_FILENO) keep_fds.push_back(logs::logFd());
if (ipc_fd >= 0) keep_fds.push_back(ipc_fd);
if (nsj->njc.exec_bin().exec_fd() && nsj->exec_fd >= 0) {
keep_fds.push_back(nsj->exec_fd);
}
/* User-requested passthrough FDs */
for (const auto fd : nsj->openfds) {
if (fd >= 0) keep_fds.push_back(fd);
}
/* Sort and deduplicate to safely iterate through the gaps */
std::sort(keep_fds.begin(), keep_fds.end());
keep_fds.erase(std::unique(keep_fds.begin(), keep_fds.end()), keep_fds.end());
unsigned int range_start = 0;
for (unsigned int target_fd : keep_fds) {
/* If there is a gap between the start of our range and the target FD, close the gap
*/
if (target_fd > range_start) {
if (util::syscall(__NR_close_range, range_start, target_fd - 1, 0) == -1) {
PLOG_E("close_range(%u, %u, 0)", range_start, target_fd - 1);
return false;
}
}
/* Advance the start of the next range to be immediately after our target FD */
range_start = target_fd + 1;
}
/* Finally, close all remaining file descriptors from the last target FD up to the system
* max */
if (range_start < ~0U) {
if (util::syscall(__NR_close_range, range_start, ~0U, 0) == -1) {
PLOG_E("close_range(%u, ~0U, 0)", range_start);
return false;
}
}
return true;
}
bool setupFD(nsj_t* nsj, int fd_in, int fd_out, int fd_err, int ipc_fd) {
if (nsj->njc.mode() == nsjail::Mode::LISTEN) {
util::detachFromTTY();
}
if (nsj->njc.stderr_to_null()) {
LOG_D("Redirecting fd=2 (STDERR_FILENO) to /dev/null");
if ((fd_err = TEMP_FAILURE_RETRY(open("/dev/null", O_RDWR))) == -1) {
PLOG_E("open('/dev/null', O_RDWR");
if ((fd_err = TEMP_FAILURE_RETRY(open("/dev/null", O_RDWR | O_CLOEXEC))) == -1) {
PLOG_E("open('/dev/null', O_RDWR | O_CLOEXEC");
return false;
}
}
if (nsj->njc.silent()) {
LOG_D("Redirecting fd=0-2 (STDIN/OUT/ERR_FILENO) to /dev/null");
if (TEMP_FAILURE_RETRY(fd_in = fd_out = fd_err = open("/dev/null", O_RDWR)) == -1) {
PLOG_E("open('/dev/null', O_RDWR)");
if (TEMP_FAILURE_RETRY(
fd_in = fd_out = fd_err = open("/dev/null", O_RDWR | O_CLOEXEC)) == -1) {
PLOG_E("open('/dev/null', O_RDWR | O_CLOEXEC)");
return false;
}
}
@@ -362,6 +368,9 @@ bool setupFD(nsj_t* nsj, int fd_in, int fd_out, int fd_err) {
PLOG_E("dup2(%d, STDERR_FILENO)", fd_err);
return false;
}
if (!contain::containCloseFDs(nsj, ipc_fd)) {
return false;
}
return true;
}
@@ -373,8 +382,7 @@ bool containProc(nsj_t* nsj) {
RETURN_ON_FAILURE(containInitUtsNs(nsj));
RETURN_ON_FAILURE(containInitCgroupNs());
RETURN_ON_FAILURE(containDropPrivs(nsj));
;
/* */
/* As non-root */
RETURN_ON_FAILURE(containCPU(nsj));
RETURN_ON_FAILURE(containTSC(nsj));

2
contain.h
View File

@@ -28,7 +28,7 @@
namespace contain {
bool setupFD(nsj_t* nsj, int fd_in, int fd_out, int fd_err);
bool setupFD(nsj_t* nsj, int fd_in, int fd_out, int fd_err, int ipc_fd);
bool containProc(nsj_t* nsj);
} // namespace contain

4
logs.cc
View File

@@ -110,9 +110,9 @@ void logMsg(enum llevel_t ll, const char* fn, int ln, bool perr, const char* fmt
return;
}
char strerr[512];
std::string strerr;
if (perr) {
snprintf(strerr, sizeof(strerr), "%s", strerror(errno));
strerr = strerror(errno);
}
struct {
const char* const descr;

2
missing_defs.h
View File

@@ -1,6 +1,6 @@
/*
nsjail - missing definitions used across the code
nsjail - missing syscall/constant/struct definitions
-----------------------------------------
Copyright 2026 Google Inc. All Rights Reserved.

10
mnt.cc
View File

@@ -47,6 +47,7 @@
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt_legacy.h"
#include "mnt_newapi.h"
#include "subproc.h"
@@ -362,17 +363,20 @@ bool initNs(nsj_t* nsj) {
return initNsInternal(nsj);
}
pid_t pid = subproc::cloneProc(CLONE_FS, SIGCHLD);
pid_t pid = subproc::cloneProcNoPidfd(CLONE_FS, SIGCHLD);
if (pid == -1) {
return false;
}
if (pid == 0) {
exit(initNsInternal(nsj) ? 0 : 0xff);
_exit(initNsInternal(nsj) ? 0 : 0xff);
}
int status;
while (wait4(pid, &status, 0, NULL) != pid);
if (TEMP_FAILURE_RETRY(waitpid(pid, &status, 0)) == -1) {
PLOG_W("waitpid(pid=%d)", pid);
return false;
}
if (WIFEXITED(status) && WEXITSTATUS(status) == 0) {
return true;
}

21
mnt.h
View File

@@ -29,28 +29,9 @@
#include <string>
#include "missing_defs.h"
#include "nsjail.h"
#if !defined(MS_NOSYMFOLLOW)
#define MS_NOSYMFOLLOW 256
#endif /* if !defined(MS_NOSYMFOLLOW) */
#if !defined(MS_LAZYTIME)
#define MS_LAZYTIME (1 << 25)
#endif /* if !defined(MS_LAZYTIME) */
#if !defined(MS_ACTIVE)
#define MS_ACTIVE (1 << 30)
#endif /* if !defined(MS_ACTIVE) */
#if !defined(MS_NOUSER)
#define MS_NOUSER (1 << 31)
#endif /* if !defined(MS_NOUSER) */
#if !defined(ST_NOSYMFOLLOW)
#define ST_NOSYMFOLLOW 8192
#endif /* if !defined(ST_NOSYMFOLLOW) */
namespace mnt {
typedef enum {

1
mnt_legacy.cc
View File

@@ -37,6 +37,7 @@
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt.h"
#include "util.h"

62
mnt_newapi.cc
View File

@@ -21,55 +21,14 @@
#include "mnt_newapi.h"
#include <dirent.h>
#include <errno.h>
#include <fcntl.h>
#include <inttypes.h>
// clang-format off
/* https://github.com/google/nsjail/issues/250 */
#include <sys/mount.h>
// clang-format on
#include <linux/mount.h>
#include <sys/syscall.h>
#include "logs.h"
#include "util.h"
/*
* Compile-time feature detection for the new mount API.
* Requires kernel headers with fsopen/fsconfig/fsmount/move_mount support.
*/
#if defined(__NR_fsopen) && defined(__NR_fsconfig) && defined(__NR_fsmount) && \
defined(__NR_move_mount) && defined(__NR_open_tree) && defined(__NR_mount_setattr) && \
defined(FSOPEN_CLOEXEC) && defined(FSMOUNT_CLOEXEC) && defined(MOVE_MOUNT_F_EMPTY_PATH) && \
defined(MOUNT_ATTR_RDONLY) && defined(MOUNT_ATTR_NOSUID) && defined(MOUNT_ATTR_NODEV) && \
defined(MOUNT_ATTR_NOEXEC) && defined(AT_EMPTY_PATH) && defined(AT_RECURSIVE)
#define MNT_NEWAPI_SUPPORTED 1
#endif
#if !defined(MNT_NEWAPI_SUPPORTED)
namespace mnt {
namespace newapi {
bool isAvailable() {
LOG_W("New mount API unavailable: missing compile-time support");
return false;
}
bool remountPt(mnt::mount_t&) {
return false;
}
std::unique_ptr<std::string> buildMountTree(nsj_t*, std::vector<mnt::mount_t>*) {
return nullptr;
}
} // namespace newapi
} // namespace mnt
#else /* MNT_NEWAPI_SUPPORTED */
#include <dirent.h>
#include <errno.h>
#include <inttypes.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <unistd.h>
@@ -78,8 +37,11 @@ std::unique_ptr<std::string> buildMountTree(nsj_t*, std::vector<mnt::mount_t>*)
#include <string>
#include <vector>
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "mnt.h"
#include "util.h"
namespace mnt {
namespace newapi {
@@ -345,8 +307,8 @@ static bool mountDynamicContentAt(mount_t* mpt, int root_fd, const char* rel_dst
return false;
}
int mnt_fd =
syscall(__NR_open_tree, root_fd, src_rel.c_str(), OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC);
int mnt_fd = util::syscall(__NR_open_tree, (uintptr_t)root_fd, (uintptr_t)src_rel.c_str(),
(uintptr_t)(OPEN_TREE_CLONE | OPEN_TREE_CLOEXEC));
if (mnt_fd < 0) {
PLOG_W("open_tree('%s')", src_rel.c_str());
unlinkat(root_fd, src_rel.c_str(), 0);
@@ -370,7 +332,8 @@ static bool mountDynamicContentAt(mount_t* mpt, int root_fd, const char* rel_dst
PLOG_W("unlinkat(root_fd, '%s')", src_rel.c_str());
}
mpt->fd = syscall(__NR_open_tree, root_fd, rel_dst, (unsigned int)OPEN_TREE_CLOEXEC);
mpt->fd = util::syscall(__NR_open_tree, (uintptr_t)root_fd, (uintptr_t)rel_dst,
(uintptr_t)(unsigned int)OPEN_TREE_CLOEXEC);
if (mpt->fd < 0) {
PLOG_W("open_tree(root_fd, '%s')", rel_dst);
return false;
@@ -386,7 +349,8 @@ static bool doBindMountAt(mount_t* mpt, int root_fd, const char* rel_dst) {
}
LOG_D("open_tree('%s', flags=0x%x)", mpt->src.c_str(), flags);
int mnt_fd = syscall(__NR_open_tree, AT_FDCWD, mpt->src.c_str(), flags);
int mnt_fd = util::syscall(
__NR_open_tree, (uintptr_t)AT_FDCWD, (uintptr_t)mpt->src.c_str(), (uintptr_t)flags);
if (mnt_fd < 0) {
PLOG_W("open_tree('%s')", mpt->src.c_str());
return false;
@@ -644,5 +608,3 @@ std::unique_ptr<std::string> buildMountTree(nsj_t* nsj, std::vector<mnt::mount_t
} // namespace newapi
} // namespace mnt
#endif /* MNT_NEWAPI_SUPPORTED */

699
monitor.cc Normal file
View File

@@ -0,0 +1,699 @@
/*
* nsjail - per-child monitor thread
* -----------------------------------------
*
* Each sandboxed child gets a dedicated monitor thread with its own
* epoll-based event loop. The thread multiplexes:
*
* pidfd -- child death notification
* ipc_fd -- parent↔child IPC socketpair
* sockproxy -- stdin/stdout splice proxy (listen mode only)
* unotify -- seccomp user notifications
* nstun -- TUN-based networking
*
* Lifecycle:
* 1. Receive setup FDs from child via IPC (tap, unotify, etc.)
* 2. Acknowledge child with MSG_TAG_READY_H2J
* 3. Enter epoll loop
* 4. On child death (pidfd) -> cleanUpAndExit()
* 5. On setup error -> killAndExit()
*/
#include "monitor.h"
#include <errno.h>
#include <fcntl.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <sys/epoll.h>
#include <sys/prctl.h>
#include <unistd.h>
#include <unordered_map>
#include <vector>
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "net.h"
#include "nsjail.h"
#include "nstun/nstun.h"
#include "sockproxy/sockproxy.h"
#include "subproc.h"
#include "unotify/unotify.h"
#include "util.h"
namespace monitor {
/* --- pipe bundle for listen-mode proxy ----------------- */
struct ProxyPipes {
int child_in;
int child_out;
int parent_in;
int parent_out;
int connfd;
void closeAll() {
if (child_in >= 0) {
close(child_in);
}
if (child_out >= 0) {
close(child_out);
}
if (parent_in >= 0) {
close(parent_in);
}
if (parent_out >= 0) {
close(parent_out);
}
if (connfd >= 0) {
close(connfd);
}
child_in = child_out = parent_in = parent_out = connfd = -1;
}
};
/* --- epoll handler types ------------------------------- */
struct fdHandler_t {
fdCb_t cb;
void* data;
};
/* --- per-thread context -------------------------------- */
constexpr size_t MAX_EVENTS = 64;
struct ThreadCtx {
/* child identity */
nsj_t* nsj = nullptr;
pid_t pid = -1;
int pidfd = -1;
time_t start_time = 0;
char remote_txt[128];
/* proxy pipe FDs (listen mode only, nullptr in standalone) */
ProxyPipes* pipes = nullptr;
/* epoll state */
int epoll_fd = -1;
std::unordered_map<int, fdHandler_t> fd_handlers;
std::vector<periodicCb_t> periodics;
bool stop_requested = false;
};
static thread_local ThreadCtx current_ctx;
bool addFd(int fd, uint32_t events, fdCb_t cb, void* data) {
if (fd < 0) {
return false;
}
current_ctx.fd_handlers[fd] = {.cb = cb, .data = data};
struct epoll_event ev = {
.events = events,
.data = {.fd = fd},
};
if (epoll_ctl(current_ctx.epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) {
PLOG_W("epoll_ctl(EPOLL_CTL_ADD, fd=%d)", fd);
current_ctx.fd_handlers.erase(fd);
return false;
}
return true;
}
bool removeFd(int fd) {
if (fd < 0) {
return false;
}
current_ctx.fd_handlers.erase(fd);
if (epoll_ctl(current_ctx.epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == -1) {
PLOG_W("epoll_ctl(EPOLL_CTL_DEL, fd=%d)", fd);
}
return true;
}
bool modFd(int fd, uint32_t events) {
if (fd < 0) {
return false;
}
struct epoll_event ev = {
.events = events,
.data = {.fd = fd},
};
if (epoll_ctl(current_ctx.epoll_fd, EPOLL_CTL_MOD, fd, &ev) == -1) {
PLOG_W("epoll_ctl(EPOLL_CTL_MOD, fd=%d)", fd);
return false;
}
return true;
}
void addPeriodic(periodicCb_t cb) {
current_ctx.periodics.push_back(cb);
}
void stop() {
current_ctx.stop_requested = true;
}
static void dispatchEvents(struct epoll_event* events, int nfds) {
for (int i = 0; i < nfds; ++i) {
int fd = events[i].data.fd;
auto it = current_ctx.fd_handlers.find(fd);
if (it != current_ctx.fd_handlers.end() && it->second.cb) {
it->second.cb(fd, events[i].events, it->second.data);
}
}
}
static void run() {
struct epoll_event events[MAX_EVENTS];
uint64_t last_periodic_ms = util::timeUsec() / 1000;
for (;;) {
uint64_t now_ms = util::timeUsec() / 1000;
int timeout_ms = current_ctx.periodics.empty()
? -1
: std::max(0, (int)(1000 - (now_ms - last_periodic_ms)));
int nfds = epoll_wait(current_ctx.epoll_fd, events, MAX_EVENTS, timeout_ms);
if (nfds == -1) {
if (errno == EINTR) {
continue;
}
PLOG_W("epoll_wait");
break;
}
dispatchEvents(events, nfds);
now_ms = util::timeUsec() / 1000;
if (!current_ctx.periodics.empty() && now_ms - last_periodic_ms >= 1000) {
last_periodic_ms = now_ms;
for (auto cb : current_ctx.periodics) {
cb();
}
}
if (current_ctx.stop_requested) {
break;
}
}
}
/* --- event loop ---------------------------------------- */
/* --- pipe setup (listen mode) -------------------------- */
static bool createProxyPipes(int connfd, ProxyPipes* pipes) {
int in[2];
int out[2];
if (pipe2(in, O_CLOEXEC) != 0) {
PLOG_W("pipe2(in)");
return false;
}
pipes->child_in = in[0];
pipes->parent_out = in[1];
if (pipe2(out, O_CLOEXEC) != 0) {
PLOG_W("pipe2(out)");
pipes->closeAll();
return false;
}
pipes->parent_in = out[0];
pipes->child_out = out[1];
/* Proxy side must be non-blocking for splice() */
if (!util::setNonBlock(pipes->parent_in) || !util::setNonBlock(pipes->parent_out)) {
pipes->closeAll();
return false;
}
pipes->connfd = connfd;
return true;
}
/* --- teardown ------------------------------------------ */
static void timeoutCb() {
subproc::checkTimeouts(current_ctx.nsj, current_ctx.pid, current_ctx.start_time,
current_ctx.remote_txt, current_ctx.pidfd);
nstun_periodic();
}
static void cleanUpAndExit() {
nstun_destroy_parent();
unotify::stop();
sockproxy::stop();
if (current_ctx.pipes) {
current_ctx.pipes->closeAll();
}
if (current_ctx.epoll_fd != -1) {
close(current_ctx.epoll_fd);
current_ctx.epoll_fd = -1;
}
current_ctx.periodics.clear();
current_ctx.fd_handlers.clear();
current_ctx.stop_requested = false;
}
static void killAndExit() {
if (current_ctx.pidfd >= 0) {
util::syscall(__NR_pidfd_send_signal, current_ctx.pidfd, SIGKILL, 0, 0);
}
cleanUpAndExit();
}
/* --- epoll callbacks ----------------------------------- */
/*
* pidfd became readable -> child is already dead.
* Just signal the loop to exit; cleanUpAndExit() skips the kill.
*/
static void pidfdCb(int /* fd */, uint32_t /* events */, void* /* data */) {
LOG_D("Child died (pidfd event), exiting monitor thread");
current_ctx.stop_requested = true;
}
/*
* IPC socketpair events:
* EPOLLIN -> message from child (e.g. execve failure)
* EPOLLRDHUP -> child closed its end (normal after execve)
*/
static void ipcFdCb(int ipc_fd, uint32_t events, void* /* data */) {
if (events & EPOLLIN) {
uint32_t id = 0;
errno = 0;
if (util::recvMsg(ipc_fd, &id, nullptr)) {
LOG_D("IPC msg 0x%08x from fd=%d", id, ipc_fd);
if (id == monitor::MSG_TAG_ERROR) {
LOG_W("Child reported error (execve failed)");
current_ctx.stop_requested = true;
}
} else {
if (errno != 0 && errno != EAGAIN && errno != EWOULDBLOCK) {
PLOG_W("recvMsg from ipc_fd failed unexpectedly");
}
}
}
if (events & (EPOLLRDHUP | EPOLLHUP | EPOLLERR)) {
LOG_D("ipc_fd=%d closed (child reached execve), unregistering", ipc_fd);
removeFd(ipc_fd);
close(ipc_fd);
}
}
/* --- IPC setup handshake ------------------------------- */
/*
* Receive setup FDs from the child (tap, unotify, etc.) until
* MSG_TAG_READY_J2H signals that the child is ready to proceed.
*
* Returns true on success, false on error (caller should killAndExit).
*/
static bool receiveChildFds(int ipc_fd, nsj_t* nsj, pid_t pid) {
for (;;) {
struct pollfd pfd = {
.fd = ipc_fd,
.events = POLLIN,
.revents = 0,
};
struct timespec ts = {
.tv_sec = 10,
.tv_nsec = 0,
};
int res = ppoll(&pfd, 1, &ts, nullptr);
if (res == -1) {
if (errno == EINTR) {
continue;
}
PLOG_W("ppoll(ipc_fd)");
return false;
}
if (res == 0) {
LOG_W("Timeout waiting for setup FDs from child");
return false;
}
uint32_t id = 0;
int fd = -1;
if (!util::recvMsg(ipc_fd, &id, &fd)) {
LOG_W("Failed to receive IPC message from child");
return false;
}
switch (id) {
case monitor::MSG_TAG_READY_J2H:
if (fd >= 0) {
close(fd);
}
return true;
case monitor::MSG_TAG_TAP:
if (!nstun_init_parent(fd, nsj, pid)) {
LOG_W("nstun_init_parent failed");
if (fd >= 0) {
close(fd);
}
return false;
}
break;
case monitor::MSG_TAG_UNOTIFY:
LOG_D("Received unotif_fd=%d from child", fd);
if (!unotify::start(nsj, fd)) {
LOG_W("Failed to start unotify");
if (fd >= 0) {
close(fd);
}
return false;
}
break;
case monitor::MSG_TAG_ERROR:
LOG_W("Child failed to launch");
if (fd >= 0) {
close(fd);
}
return false;
default:
LOG_W("Unknown IPC message type 0x%08x from child", id);
if (fd >= 0) {
close(fd);
}
return false;
}
}
}
static void proxyCloseCb(void* /* data */) {
ThreadCtx* ctx = &current_ctx;
LOG_D("Proxy closed, killing child process (pid=%d)", ctx->pid);
util::syscall(__NR_pidfd_send_signal, ctx->pidfd, SIGKILL, 0, 0);
monitor::stop();
}
/* --- monitor thread ------------------------------------ */
struct MonitorArgs {
nsj_t* nsj;
pid_t pid;
int ipc_fd;
int pidfd;
ProxyPipes pipes; /* copied by value; connfd < 0 means no proxy */
time_t start_time;
char remote_txt[128];
};
static void monitorThread(MonitorArgs args) {
/* -- Thread identity -- */
char name[16];
snprintf(name, sizeof(name), "nsj-mon-%d", args.pid);
prctl(PR_SET_NAME, name, 0, 0, 0);
sigset_t set;
sigemptyset(&set);
for (int sig : nssigs) {
sigaddset(&set, sig);
}
pthread_sigmask(SIG_BLOCK, &set, nullptr);
/* -- Initialize thread-local context -- */
current_ctx = ThreadCtx();
current_ctx.nsj = args.nsj;
current_ctx.pid = args.pid;
current_ctx.pidfd = args.pidfd;
current_ctx.pipes = (args.pipes.connfd >= 0) ? &args.pipes : nullptr;
current_ctx.start_time = args.start_time;
snprintf(current_ctx.remote_txt, sizeof(current_ctx.remote_txt), "%s", args.remote_txt);
LOG_I("Monitor thread started for pid=%d (ipc_fd=%d, pidfd=%d) '%s'", (int)args.pid,
args.ipc_fd, args.pidfd, current_ctx.remote_txt);
/* -- Create epoll instance -- */
current_ctx.epoll_fd = epoll_create1(EPOLL_CLOEXEC);
if (current_ctx.epoll_fd == -1) {
LOG_W("epoll_create1 failed");
killAndExit();
return;
}
/* -- Phase 1: receive setup FDs from child -- */
if (!receiveChildFds(args.ipc_fd, args.nsj, args.pid)) {
killAndExit();
return;
}
/* -- Phase 2: acknowledge child -- */
if (!util::sendMsg(args.ipc_fd, monitor::MSG_TAG_READY_H2J)) {
LOG_W("Failed to send READY to child");
killAndExit();
return;
}
/* -- Phase 3: register epoll sources -- */
if (!addFd(args.pidfd, EPOLLIN, pidfdCb, nullptr)) {
killAndExit();
return;
}
if (current_ctx.pipes) {
if (!sockproxy::start(&current_ctx.pipes->connfd, &current_ctx.pipes->parent_out,
&current_ctx.pipes->parent_in, proxyCloseCb, nullptr)) {
killAndExit();
return;
}
}
if (!util::setNonBlock(args.ipc_fd) ||
!addFd(args.ipc_fd, EPOLLIN | EPOLLRDHUP, ipcFdCb, nullptr)) {
killAndExit();
return;
}
/* -- Phase 4: run event loop -- */
addPeriodic(timeoutCb);
run();
/* -- Phase 5: kill child (harmless if already dead) and clean up -- */
util::syscall(__NR_pidfd_send_signal, current_ctx.pidfd, SIGKILL, 0, 0);
cleanUpAndExit();
}
/* --- thread launcher ----------------------------------- */
static void startMonitorThread(
nsj_t* nsj, pid_t pid, int ipc_fd, int pidfd, ProxyPipes* pipes, std::thread* thread_out) {
MonitorArgs args = {
.nsj = nsj,
.pid = pid,
.ipc_fd = ipc_fd,
.pidfd = pidfd,
.pipes = pipes ? *pipes : ProxyPipes{-1, -1, -1, -1, -1},
.start_time = 0,
.remote_txt = {},
};
/* Copy child metadata on the main thread to avoid racing with reapAll */
auto it = nsj->pids.find(pid);
if (it != nsj->pids.end()) {
args.start_time = it->second.start;
snprintf(args.remote_txt, sizeof(args.remote_txt), "%s",
it->second.remote_txt.c_str());
}
*thread_out = std::thread(monitorThread, args);
}
/* --- signal handling helpers --------------------------- */
/*
* Handle a fatal signal during the main-thread poll loop.
* First signal: forward to children (or SIGKILL), close listen socket.
* After 2s: escalate to SIGKILL.
* Returns true if we should break out of the outer loop.
*/
static bool handleShutdownSignal(nsj_t* nsj, int sig, time_t* shutdown_start, int* listenfd) {
if (*shutdown_start == 0) {
*shutdown_start = time(nullptr);
subproc::killAll(nsj, nsj->njc.forward_signals() ? sig : SIGKILL);
logs::logStop(sig);
if (listenfd && *listenfd >= 0) {
close(*listenfd);
*listenfd = -1;
}
} else if (time(nullptr) - *shutdown_start >= 2) {
LOG_W("Processes did not exit after 2s, escalating to SIGKILL");
subproc::killAll(nsj, SIGKILL);
}
subproc::reapAll(nsj);
return (subproc::countProc(nsj) == 0);
}
/* --- listen mode (-Ml) -------------------------------- */
int runListenMode(nsj_t* nsj) {
int listenfd = net::getRecvSocket(nsj);
if (listenfd == -1) {
return EXIT_FAILURE;
}
time_t shutdown_start = 0;
for (;;) {
struct pollfd pfd = {
.fd = listenfd,
.events = POLLIN,
.revents = 0,
};
struct timespec ts = {
.tv_sec = 1,
.tv_nsec = 0,
};
int res = ppoll(&pfd, 1, &ts, nullptr);
if (res == -1 && errno != EINTR) {
PLOG_W("ppoll");
break;
}
/* -- Shutdown path -- */
int sig = nsjail::getSigFatal();
if (sig > 0) {
if (handleShutdownSignal(nsj, sig, &shutdown_start, &listenfd)) {
break;
}
continue;
}
/* -- Accept new connection -- */
if (res > 0 && listenfd >= 0 && (pfd.revents & POLLIN)) {
int connfd = net::acceptConn(listenfd);
if (connfd >= 0) {
ProxyPipes pipes = {
.child_in = -1,
.child_out = -1,
.parent_in = -1,
.parent_out = -1,
.connfd = -1,
};
if (createProxyPipes(connfd, &pipes)) {
int pidfd = -1;
int ipc_fd = -1;
pid_t pid = subproc::runChild(nsj, connfd, pipes.child_in,
pipes.child_out, pipes.child_out, &pidfd, &ipc_fd);
if (pid <= 0) {
pipes.closeAll();
} else {
/* Parent doesn't need these */
close(pipes.child_in);
pipes.child_in = -1;
close(pipes.child_out);
pipes.child_out = -1;
startMonitorThread(nsj, pid, ipc_fd, pidfd, &pipes,
&nsj->pids[pid].thread);
}
} else {
close(connfd);
}
}
}
/* -- Housekeeping -- */
subproc::reapAll(nsj);
if (nsjail::shouldShowProc()) {
nsjail::clearShowProc();
subproc::displayProc(nsj);
}
}
if (listenfd >= 0) {
close(listenfd);
}
return EXIT_SUCCESS;
}
/* --- standalone mode (-Mo / -Me) ----------------------- */
int runStandaloneMode(nsj_t* nsj) {
for (;;) {
int pidfd = -1;
int ipc_fd = -1;
pid_t pid = subproc::runChild(nsj, /* netfd= */ -1, STDIN_FILENO, STDOUT_FILENO,
STDERR_FILENO, &pidfd, &ipc_fd);
if (pid == -1) {
LOG_E("Couldn't launch the child process");
return 0xff;
}
time_t start_time = nsj->pids[pid].start;
startMonitorThread(nsj, pid, ipc_fd, pidfd, nullptr, &nsj->pids[pid].thread);
/* -- Wait for child on main thread -- */
time_t shutdown_start = 0;
for (;;) {
struct pollfd pfd = {
.fd = pidfd,
.events = POLLIN,
.revents = 0,
};
struct timespec ts = {
.tv_sec = 1,
.tv_nsec = 0,
};
int res = ppoll(&pfd, 1, &ts, nullptr);
if (res == -1 && errno != EINTR) {
PLOG_W("ppoll");
break;
}
/* -- Shutdown path -- */
int sig = nsjail::getSigFatal();
if (sig > 0) {
handleShutdownSignal(nsj, sig, &shutdown_start, nullptr);
}
if (res > 0 && (pfd.revents & (POLLIN | POLLERR | POLLHUP))) {
break;
}
if (nsjail::shouldShowProc()) {
nsjail::clearShowProc();
subproc::displayProc(nsj);
}
}
/* Main thread reaps the child and joins its monitor thread */
subproc::reapProc(nsj, pid, true);
if (subproc::countProc(nsj) == 0) {
if (nsj->njc.mode() == nsjail::Mode::ONCE) {
return nsj->exit_status;
}
time_t now = time(nullptr);
if (now - start_time < 1) {
LOG_I("Child exited too quickly, rate-limiting respawn");
struct timespec ts = {.tv_sec = 1, .tv_nsec = 0};
ppoll(nullptr, 0, &ts, nullptr);
}
/* Daemon mode: check for pending fatal signals before respawning.
* Without this, a SIGINT arriving during reapProc could spawn an
* unwanted new child (Infinite Daemon Loop Law, goal.md #4). */
if (nsjail::getSigFatal() > 0) {
return nsj->exit_status;
}
}
}
/* not reached */
}
} // namespace monitor

50
monitor.h Normal file
View File

@@ -0,0 +1,50 @@
/*
* nsjail - epoll-based event loop interface
* -----------------------------------------
*
* Public API for the per-child monitor thread and its single-threaded
* event loop. Each child process gets one monitor thread that
* multiplexes I/O from pidfd, unotify, sockproxy, and nstun sources.
*/
#ifndef NS_MONITOR_H
#define NS_MONITOR_H
#include <stdint.h>
struct nsj_t;
namespace monitor {
/* --- IPC message tags ---------------------------------- */
constexpr uint32_t quad(char a, char b, char c, char d) {
return (static_cast<uint32_t>(a)) | (static_cast<uint32_t>(b) << 8) |
(static_cast<uint32_t>(c) << 16) | (static_cast<uint32_t>(d) << 24);
}
constexpr uint32_t MSG_TAG_TAP = quad('T', 'A', 'P', 'F');
constexpr uint32_t MSG_TAG_UNOTIFY = quad('S', 'E', 'C', 'F');
constexpr uint32_t MSG_TAG_READY_J2H = quad('R', 'D', 'J', 'H');
constexpr uint32_t MSG_TAG_READY_H2J = quad('R', 'D', 'H', 'J');
constexpr uint32_t MSG_TAG_ERROR = quad('E', 'R', 'R', 'F');
/* --- epoll loop API ------------------------------------ */
typedef void (*fdCb_t)(int fd, uint32_t events, void* data);
typedef void (*periodicCb_t)();
bool addFd(int fd, uint32_t events, fdCb_t cb, void* data);
bool removeFd(int fd);
bool modFd(int fd, uint32_t events);
void addPeriodic(periodicCb_t cb);
void stop();
/* --- entry points -------------------------------------- */
int runListenMode(nsj_t* nsj);
int runStandaloneMode(nsj_t* nsj);
} // namespace monitor
#endif /* NS_MONITOR_H */

78
net.cc
View File

@@ -30,7 +30,6 @@
#include <netinet/in.h>
#include <netinet/ip6.h>
#include <netinet/tcp.h>
#include <netlink/route/nexthop.h>
#include <netlink/route/route.h>
#include <netlink/route/rule.h>
#include <stdint.h>
@@ -55,7 +54,9 @@
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "nstun/nstun.h"
#include "subproc.h"
#include "util.h"
#define STR_(x) #x
@@ -136,13 +137,15 @@ static bool cloneIface(nsj_t* nsj, struct nl_sock* sk, struct nl_cache* link_cac
LOG_E("rtnl_link_macvlan_alloc()");
return false;
}
defer {
rtnl_link_put(rmv);
};
int err;
int master_index = rtnl_link_name2i(link_cache, nsj->njc.macvlan_iface().c_str());
if (!master_index) {
LOG_E("rtnl_link_name2i(): Did not find '%s' interface",
nsj->njc.macvlan_iface().c_str());
rtnl_link_put(rmv);
return false;
}
@@ -170,29 +173,32 @@ static bool cloneIface(nsj_t* nsj, struct nl_sock* sk, struct nl_cache* link_cac
if ((err = rtnl_link_add(sk, rmv, NLM_F_CREATE)) < 0) {
LOG_E("rtnl_link_add(name:'%s' link:'%s'): %s", IFACE_NAME,
nsj->njc.macvlan_iface().c_str(), nl_geterror(err));
rtnl_link_put(rmv);
return false;
}
rtnl_link_put(rmv);
return true;
}
static bool moveToNs(
const std::string& iface, struct nl_sock* sk, struct nl_cache* link_cache, pid_t pid) {
LOG_D("Moving interface '%s' into netns=%d", iface.c_str(), (int)pid);
struct rtnl_link* orig_link = rtnl_link_get_by_name(link_cache, iface.c_str());
if (!orig_link) {
LOG_E("Couldn't find interface '%s'", iface.c_str());
return false;
}
defer {
rtnl_link_put(orig_link);
};
struct rtnl_link* new_link = rtnl_link_alloc();
if (!new_link) {
LOG_E("Couldn't allocate new link");
rtnl_link_put(orig_link);
return false;
}
defer {
rtnl_link_put(new_link);
};
rtnl_link_set_ns_pid(new_link, pid);
@@ -200,13 +206,9 @@ static bool moveToNs(
if (err < 0) {
LOG_E("rtnl_link_change(): set NS of interface '%s' to pid=%d: %s", iface.c_str(),
(int)pid, nl_geterror(err));
rtnl_link_put(new_link);
rtnl_link_put(orig_link);
return false;
}
rtnl_link_put(new_link);
rtnl_link_put(orig_link);
return true;
}
@@ -324,9 +326,9 @@ static void pastaProcess(nsj_t* nsj, int pid, int err_pipe) {
argv.push_back(pid_str.c_str());
argv.push_back(nullptr);
int nullfd = TEMP_FAILURE_RETRY(open("/dev/null", O_RDWR));
int nullfd = TEMP_FAILURE_RETRY(open("/dev/null", O_RDWR | O_CLOEXEC));
if (nullfd == -1) {
PLOG_E("Cannot open '/dev/null' - O_RDWR");
PLOG_E("Cannot open '/dev/null' - O_RDWR | O_CLOEXEC");
_exit(EXIT_FAILURE);
}
if (TEMP_FAILURE_RETRY(dup2(nullfd, STDIN_FILENO)) == -1) {
@@ -379,11 +381,10 @@ static bool spawnPasta(nsj_t* nsj, int pid) {
return false;
}
pid_t ppid = fork();
pid_t ppid = subproc::cloneProcNoPidfd(0, SIGCHLD);
if (ppid == -1) {
close(sv[0]);
close(sv[1]);
PLOG_E("fork()");
return false;
}
@@ -398,7 +399,7 @@ static bool spawnPasta(nsj_t* nsj, int pid) {
if (util::readFromFd(sv[0], &err, sizeof(err)) > 0) {
close(sv[0]);
LOG_E("Pasta execution failed, error: %s", strerror(err));
while (waitpid(ppid, nullptr, 0) == -1 && errno == EINTR);
TEMP_FAILURE_RETRY(waitpid(ppid, nullptr, 0));
return false;
}
@@ -408,7 +409,7 @@ static bool spawnPasta(nsj_t* nsj, int pid) {
return true;
}
bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
bool initParent(nsj_t* nsj, pid_t pid, int ipc_fd) {
if (nsj->njc.has_user_net()) {
if (!nsj->njc.clone_newnet()) {
LOG_E("Support for User-Mode Networking requested but CLONE_NEWNET "
@@ -416,10 +417,7 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
return false;
}
if (nsj->njc.user_net().backend() == nsjail::NsJailConfig_UserNet_Backend_NSTUN) {
if (!nstun_init_parent(pipefd, nsj)) {
LOG_E("nstun_init_parent() failed");
return false;
}
/* Context creation is delegated to the monitor thread */
} else if (nsj->njc.user_net().backend() ==
nsjail::NsJailConfig_UserNet_Backend_PASTA &&
nsj->njc.user_net().has_pasta()) {
@@ -457,12 +455,10 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
for (const auto& iface : nsj->njc.iface_own()) {
if (!moveToNs(iface, sk, link_cache, pid)) {
nl_cache_free(link_cache);
return false;
}
}
if (!nsj->njc.macvlan_iface().empty() && !cloneIface(nsj, sk, link_cache, pid)) {
nl_cache_free(link_cache);
return false;
}
@@ -535,18 +531,15 @@ int getRecvSocket(const nsj_t* nsj) {
return -1;
}
int sockfd = socket(AF_INET6, SOCK_STREAM, 0);
int sockfd = socket(AF_INET6, SOCK_STREAM | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (sockfd == -1) {
PLOG_E("socket(AF_INET6)");
return -1;
}
if (fcntl(sockfd, F_SETFL, O_NONBLOCK)) {
PLOG_E("fcntl(%d, F_SETFL, O_NONBLOCK)", sockfd);
return -1;
}
int so = 1;
if (setsockopt(sockfd, SOL_SOCKET, SO_REUSEADDR, &so, sizeof(so)) == -1) {
PLOG_E("setsockopt(%d, SO_REUSEADDR)", sockfd);
close(sockfd);
return -1;
}
struct sockaddr_in6 addr = {
@@ -576,9 +569,10 @@ int getRecvSocket(const nsj_t* nsj) {
int acceptConn(int listenfd) {
struct sockaddr_in6 cli_addr = {};
socklen_t socklen = sizeof(cli_addr);
int connfd = accept4(listenfd, (struct sockaddr*)&cli_addr, &socklen, SOCK_NONBLOCK);
int connfd = TEMP_FAILURE_RETRY(
accept4(listenfd, (struct sockaddr*)&cli_addr, &socklen, SOCK_NONBLOCK | SOCK_CLOEXEC));
if (connfd == -1) {
if (errno != EINTR) {
if (errno != EINTR && errno != EAGAIN && errno != EWOULDBLOCK) {
PLOG_E("accept(%d)", listenfd);
}
return -1;
@@ -630,7 +624,7 @@ const std::string connToText(int fd, bool remote, struct sockaddr_in6* addr_or_n
}
static bool ifaceUp(const char* ifacename) {
int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_IP);
int sock = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_IP);
if (sock == -1) {
PLOG_E("socket(AF_INET, SOCK_STREAM, IPPROTO_IP)");
return false;
@@ -640,7 +634,10 @@ static bool ifaceUp(const char* ifacename) {
};
struct ifreq ifr = {};
snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", ifacename);
if (snprintf(ifr.ifr_name, IF_NAMESIZE, "%s", ifacename) >= IF_NAMESIZE) {
LOG_W("Interface name '%s' is too long", ifacename);
return false;
}
if (ioctl(sock, SIOCGIFFLAGS, &ifr) == -1) {
PLOG_E("ioctl(iface='%s', SIOCGIFFLAGS, IFF_UP)", ifacename);
@@ -662,21 +659,22 @@ static bool ifaceUp(const char* ifacename) {
static bool ifaceConfig(const std::string& iface, const std::string& ip, const std::string& mask,
const std::string& gw) {
int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_IP);
int sock = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_IP);
if (sock == -1) {
PLOG_E("socket(AF_INET, SOCK_STREAM, IPPROTO_IP)");
return false;
}
defer {
close(sock);
};
struct in_addr addr;
if (inet_pton(AF_INET, ip.c_str(), &addr) != 1) {
PLOG_E("Cannot convert '%s' into an IPv4 address", ip.c_str());
close(sock);
return false;
}
if (addr.s_addr == INADDR_ANY) {
LOG_D("IPv4 address for interface '%s' not set", iface.c_str());
close(sock);
return true;
}
@@ -687,36 +685,30 @@ static bool ifaceConfig(const std::string& iface, const std::string& ip, const s
sa->sin_addr = addr;
if (ioctl(sock, SIOCSIFADDR, &ifr) == -1) {
PLOG_E("ioctl(iface='%s', SIOCSIFADDR, '%s')", iface.c_str(), ip.c_str());
close(sock);
return false;
}
if (inet_pton(AF_INET, mask.c_str(), &addr) != 1) {
PLOG_E("Cannot convert '%s' into a IPv4 netmask", mask.c_str());
close(sock);
return false;
}
sa->sin_family = AF_INET;
sa->sin_addr = addr;
if (ioctl(sock, SIOCSIFNETMASK, &ifr) == -1) {
PLOG_E("ioctl(iface='%s', SIOCSIFNETMASK, '%s')", iface.c_str(), mask.c_str());
close(sock);
return false;
}
if (!ifaceUp(iface.c_str())) {
close(sock);
return false;
}
if (inet_pton(AF_INET, gw.c_str(), &addr) != 1) {
PLOG_E("Cannot convert '%s' into a IPv4 GW address", gw.c_str());
close(sock);
return false;
}
if (addr.s_addr == INADDR_ANY) {
LOG_D("Gateway address for '%s' is not set", iface.c_str());
close(sock);
return true;
}
@@ -738,11 +730,9 @@ static bool ifaceConfig(const std::string& iface, const std::string& ip, const s
if (ioctl(sock, SIOCADDRT, &rt) == -1) {
PLOG_E("ioctl(SIOCADDRT, '%s')", gw.c_str());
close(sock);
return false;
}
close(sock);
return true;
}
@@ -895,10 +885,10 @@ bool initNsFromChild(nsj_t* nsj) {
return true;
}
bool initChildPreSync(nsj_t* nsj, int pipefd) {
bool initChildPreSync(nsj_t* nsj, int ipc_fd) {
if (nsj->njc.has_user_net()) {
if (nsj->njc.user_net().backend() == nsjail::NsJailConfig_UserNet_Backend_NSTUN) {
if (!nstun_init_child(pipefd, nsj)) {
if (!nstun_init_child(ipc_fd, nsj)) {
LOG_E("nstun_init_child() failed");
return false;
}

5
net.h
View File

@@ -27,6 +27,7 @@
#include <string>
#include "monitor.h"
#include "nsjail.h"
namespace net {
@@ -36,8 +37,8 @@ int getRecvSocket(const nsj_t* nsj);
int acceptConn(int listenfd);
const std::string connToText(int fd, bool remote, struct sockaddr_in6* addr_or_null);
bool initNs(nsj_t* nsj);
bool initChildPreSync(nsj_t* nsj, int pipefd);
bool initParent(nsj_t* nsj, pid_t pid, int pipefd);
bool initChildPreSync(nsj_t* nsj, int ipc_fd);
bool initParent(nsj_t* nsj, pid_t pid, int ipc_fd);
} // namespace net

234
nsjail.cc
View File

@@ -22,15 +22,19 @@
#include "nsjail.h"
#include <fcntl.h>
#include <poll.h>
#if __has_include(<linux/close_range.h>)
#include <linux/close_range.h>
#endif
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/prctl.h>
#include <sys/resource.h>
#include <sys/time.h>
#include <sys/wait.h>
#include <termios.h>
#include <unistd.h>
@@ -38,22 +42,30 @@
#include <atomic>
#include <cerrno>
#include <memory>
#include <thread>
#include <vector>
#include "cgroup2.h"
#include "cmdline.h"
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "monitor.h"
#include "net.h"
#include "sandbox.h"
#include "subproc.h"
#include "unotify/unotify.h"
#include "unotify/stats.h"
#include "util.h"
namespace nsjail {
static __thread std::atomic<int> sigFatal(0);
static __thread std::atomic<bool> showProc(false);
/*
* Thread-local to the main thread. Signals are only delivered to the main thread
* (monitor threads block them), and only the main thread reads these in its poll loops.
* See "The Threading Model Comprehension Law" in goal.md.
*/
static __thread std::atomic<int> sigFatal{0};
static __thread std::atomic<bool> showProc{false};
static void sigHandler(int sig) {
if (sig == SIGALRM || sig == SIGCHLD || sig == SIGPIPE) {
@@ -88,6 +100,16 @@ static bool setSigHandler(int sig) {
return true;
}
int getSigFatal() {
return sigFatal;
}
bool shouldShowProc() {
return showProc;
}
void clearShowProc() {
showProc = false;
}
static bool setSigHandlers(void) {
for (const auto& i : nssigs) {
if (!setSigHandler(i)) {
@@ -98,7 +120,7 @@ static bool setSigHandlers(void) {
}
static bool setTimer(nsj_t* nsj) {
if (nsj->njc.mode() == nsjail::Mode::EXECVE) {
if (nsj->njc.mode() == ::nsjail::Mode::EXECVE) {
return true;
}
@@ -144,185 +166,6 @@ static bool setFDLimit() {
return true;
}
static bool pipeTraffic(nsj_t* nsj, int listenfd) {
std::vector<struct pollfd> fds;
fds.reserve(nsj->pipes.size() * 3 + 1);
for (const auto& p : nsj->pipes) {
fds.push_back({
.fd = p.sock_fd,
.events = POLLIN | POLLOUT,
.revents = 0,
});
fds.push_back({
.fd = p.pipe_in,
.events = POLLOUT,
.revents = 0,
});
fds.push_back({
.fd = p.pipe_out,
.events = POLLIN,
.revents = 0,
});
}
fds.push_back({
.fd = listenfd,
.events = POLLIN,
.revents = 0,
});
LOG_D("Waiting for fd activity");
while (poll(fds.data(), fds.size(), -1) > 0) {
if (sigFatal > 0 || showProc) {
return false;
}
if (fds.back().revents != 0) {
LOG_D("New connection ready");
return true;
}
bool cleanup = false;
for (size_t i = 0; i < fds.size() - 1; ++i) {
if (fds[i].revents & POLLIN) {
fds[i].events &= ~POLLIN;
}
if (fds[i].revents & POLLOUT) {
fds[i].events &= ~POLLOUT;
}
}
for (size_t i = 0; i < nsj->pipes.size() * 3; i += 3) {
const size_t pipe_no = i / 3;
int in, out;
const char* direction;
bool closed = false;
std::tuple<int, int, const char*> direction_map[] = {
std::make_tuple(i, i + 1, "in"),
std::make_tuple(i + 2, i, "out"),
};
for (const auto& entry : direction_map) {
std::tie(in, out, direction) = entry;
bool in_ready = (fds[in].events & POLLIN) == 0 ||
(fds[in].revents & POLLIN) == POLLIN;
bool out_ready = (fds[out].events & POLLOUT) == 0 ||
(fds[out].revents & POLLOUT) == POLLOUT;
if (in_ready && out_ready) {
LOG_D("#%zu piping data %s", pipe_no, direction);
ssize_t rv = splice(fds[in].fd, nullptr, fds[out].fd,
nullptr, 4096, SPLICE_F_NONBLOCK);
if (rv == -1 && errno != EAGAIN) {
PLOG_E("splice fd pair #%zu {%d, %d}\n", pipe_no,
fds[in].fd, fds[out].fd);
}
if (rv == 0) {
closed = true;
}
fds[in].events |= POLLIN;
fds[out].events |= POLLOUT;
}
if ((fds[in].revents & (POLLERR | POLLHUP)) != 0 ||
(fds[out].revents & (POLLERR | POLLHUP)) != 0) {
closed = true;
}
}
if (closed) {
LOG_D("#%zu connection closed", pipe_no);
cleanup = true;
close(nsj->pipes[pipe_no].sock_fd);
close(nsj->pipes[pipe_no].pipe_in);
close(nsj->pipes[pipe_no].pipe_out);
if (nsj->pipes[pipe_no].pid > 0) {
kill(nsj->pipes[pipe_no].pid, SIGKILL);
}
nsj->pipes[pipe_no] = {};
}
}
if (cleanup) {
break;
}
}
nsj->pipes.erase(
std::remove(nsj->pipes.begin(), nsj->pipes.end(), pipemap_t{}), nsj->pipes.end());
return false;
}
static int listenMode(nsj_t* nsj) {
int listenfd = net::getRecvSocket(nsj);
if (listenfd == -1) {
return EXIT_FAILURE;
}
for (;;) {
if (sigFatal > 0) {
subproc::killAndReapAll(
nsj, nsj->njc.forward_signals() ? sigFatal.load() : SIGKILL);
logs::logStop(sigFatal);
close(listenfd);
return EXIT_SUCCESS;
}
if (showProc) {
showProc = false;
subproc::displayProc(nsj);
}
if (pipeTraffic(nsj, listenfd)) {
int connfd = net::acceptConn(listenfd);
if (connfd >= 0) {
int in[2];
int out[2];
if (pipe(in) != 0 || pipe(out) != 0) {
PLOG_E("pipe");
continue;
}
pid_t pid = subproc::runChild(nsj, connfd, in[0], out[1], out[1]);
close(in[0]);
close(out[1]);
if (pid <= 0) {
close(in[1]);
close(out[0]);
close(connfd);
} else {
nsj->pipes.push_back({
.sock_fd = connfd,
.pipe_in = in[1],
.pipe_out = out[0],
.pid = pid,
});
}
}
}
subproc::reapProc(nsj);
}
}
static int standaloneMode(nsj_t* nsj) {
for (;;) {
if (subproc::runChild(
nsj, /* netfd= */ -1, STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO) == -1) {
LOG_E("Couldn't launch the child process");
return 0xff;
}
for (;;) {
int child_status = subproc::reapProc(nsj);
if (subproc::countProc(nsj) == 0) {
if (nsj->njc.mode() == nsjail::Mode::ONCE) {
return child_status;
}
break;
}
if (showProc) {
showProc = false;
subproc::displayProc(nsj);
}
if (sigFatal > 0) {
subproc::killAndReapAll(
nsj, nsj->njc.forward_signals() ? sigFatal.load() : SIGKILL);
logs::logStop(sigFatal);
return (128 + sigFatal);
}
pause();
}
}
// not reached
}
std::unique_ptr<struct termios> getTC(int fd) {
std::unique_ptr<struct termios> trm(new struct termios);
@@ -351,6 +194,19 @@ void setTC(int fd, const struct termios* trm) {
} // namespace nsjail
int main(int argc, char* argv[]) {
/*
* Hard minimum: clone3 (5.3), CLONE_PIDFD (5.4),
* CLONE_CLEAR_SIGHAND (5.5), PIDFD_NONBLOCK (5.10),
* CLOSE_RANGE_CLOEXEC (5.11).
*/
if (!util::kernelVersionAtLeast(5, 11, 0)) {
LOG_F("This version of nsjail requires Linux >= 5.11. "
"Use an earlier version of nsjail for older kernels.");
}
if (!util::kernelVersionAtLeast(6, 0, 0)) {
LOG_D("Running on a kernel older than 6.0. Consider upgrading "
"for best compatibility.");
}
std::unique_ptr<nsj_t> nsj = cmdline::parseArgs(argc, argv);
std::unique_ptr<struct termios> trm = nsjail::getTC(STDIN_FILENO);
@@ -387,15 +243,15 @@ int main(int argc, char* argv[]) {
}
int ret = 0;
if (nsj->njc.mode() == nsjail::Mode::LISTEN) {
ret = nsjail::listenMode(nsj.get());
if (nsj->njc.mode() == ::nsjail::Mode::LISTEN) {
ret = monitor::runListenMode(nsj.get());
} else {
ret = nsjail::standaloneMode(nsj.get());
ret = monitor::runStandaloneMode(nsj.get());
}
subproc::killAndReapAll(nsj.get(), SIGKILL);
subproc::killAll(nsj.get(), SIGKILL);
sandbox::closePolicy(nsj.get());
unotify::stop(nsj.get());
unotify::printStats(nsj.get());
/* Try to restore the underlying console's params in case some program has changed it */
if (!nsj->njc.daemon()) {
nsjail::setTC(STDIN_FILENO, trm.get());

3
nsjail.h
View File

@@ -34,6 +34,7 @@
#include <map>
#include <string>
#include <thread>
#include <vector>
#include "config.pb.h"
@@ -58,7 +59,7 @@ struct pids_t {
struct sockaddr_in6 remote_addr;
pid_t pasta_pid;
pthread_t monitor_tid;
std::thread thread;
};
struct idmap_t {

283
nstun/core.h
View File

@@ -5,29 +5,31 @@
#include <stdint.h>
#include <time.h>
#include <unistd.h>
#include <sys/socket.h>
#include <sys/uio.h>
#include <compare>
#include <deque>
#include <map>
#include <memory>
#include <vector>
#include "net_defs.h"
#include "nstun.h"
#include "encap.h"
namespace nstun {
constexpr size_t NSTUN_MAX_FLOWS = 1024;
constexpr size_t NSTUN_MAX_FDS = 2048;
constexpr size_t NSTUN_MAX_RULES = 128;
constexpr size_t UDP_QUEUE_PACKET_MAX = 1500;
constexpr size_t UDP_QUEUE_MAX_PACKETS = 50;
constexpr int VLEN = 64;
// Removed MemcmpLess in favor of C++20 operator<=>
/* Removed MemcmpLess in favor of C++20 operator<=> */
struct __attribute__((packed)) FlowKey4 {
uint32_t saddr4;
uint32_t daddr4;
uint16_t sport;
uint16_t dport;
auto operator<=> (const FlowKey4&) const = default;
};
struct __attribute__((packed)) FlowKey6 {
@@ -35,119 +37,128 @@ struct __attribute__((packed)) FlowKey6 {
uint8_t daddr6[IPV6_ADDR_LEN];
uint16_t sport;
uint16_t dport;
auto operator<=> (const FlowKey6&) const = default;
};
struct __attribute__((packed)) IcmpFlowKey4 {
uint32_t saddr4;
uint32_t daddr4;
uint16_t id;
auto operator<=> (const IcmpFlowKey4&) const = default;
};
struct __attribute__((packed)) IcmpFlowKey6 {
uint8_t saddr6[IPV6_ADDR_LEN];
uint8_t daddr6[IPV6_ADDR_LEN];
uint16_t id;
auto operator<=> (const IcmpFlowKey6&) const = default;
};
struct Context;
class Flow {
public:
time_t last_active = 0;
bool is_ipv6 = false;
virtual ~Flow() = default;
virtual void handle_host_event(Context* ctx, int fd, uint32_t events) = 0;
virtual void periodic_check(Context* ctx, time_t now) {}
virtual bool is_stale(time_t now) const = 0;
virtual void destroy(Context* ctx) = 0;
};
enum class FlowType { TCP, UDP, ICMP };
enum class ProxyMode : uint8_t { NONE, SOCKS5, HTTP_CONNECT };
enum class UdpSocks5State {
ESTABLISHED, /* Direct or SOCKS5 ready */
SOCKS5_GREETING, /* Sent SOCKS5 greeting, awaiting auth reply */
SOCKS5_ASSOCIATE, /* Sent UDP ASSOCIATE, awaiting BND addr */
TCP_CONNECTING, /* TCP connect() to SOCKS5 proxy in progress */
enum UdpSocks5State {
UDP_S5_ESTABLISHED, /* Direct or SOCKS5 ready */
UDP_S5_GREETING, /* Sent SOCKS5 greeting, awaiting auth reply */
UDP_S5_ASSOCIATE, /* Sent UDP ASSOCIATE, awaiting BND addr */
UDP_S5_TCP_CONNECTING, /* TCP connect() to SOCKS5 proxy in progress */
};
struct UdpFlow : public Flow {
int host_fd = -1;
int tcp_fd = -1; /* For SOCKS5 UDP associate */
enum class TcpState {
SYN_SENT, /* Host connecting to destination */
SOCKS5_INIT, /* Sent SOCKS5 greeting, awaiting auth reply */
SOCKS5_CONNECTING, /* Sent SOCKS5 CONNECT request, awaiting response */
HTTP_CONNECT_WAIT, /* Sent HTTP CONNECT, awaiting proxy 200 reply */
ESTABLISHED,
FIN_WAIT_1,
FIN_WAIT_2,
CLOSING,
TIME_WAIT,
CLOSE_WAIT,
};
struct FlowHeader {
bool active;
FlowType type;
time_t last_active;
bool is_ipv6;
int host_fd;
union {
FlowKey4 key4;
FlowKey6 key6;
IcmpFlowKey4 icmp_key4;
IcmpFlowKey6 icmp_key6;
};
bool is_redirected = false;
/*
* Original destination (before redirect/SOCKS5 rewrite).
* Used when forwarding host replies back to the guest.
*/
union {
uint32_t orig_dest_ip4;
uint8_t orig_dest_ip6[16];
};
uint16_t orig_dest_port = 0;
/*
* Redirect destination - stored at flow creation so the forwarding
* path never needs to re-evaluate the rule. For SOCKS5 flows this
* is the proxy address; for plain REDIRECT rules it is the target.
*/
uint32_t redirect_ip4 = 0;
uint8_t redirect_ip6[IPV6_ADDR_LEN] = {};
uint16_t redirect_port = 0;
bool use_socks5 = false;
UdpSocks5State state = UdpSocks5State::ESTABLISHED;
uint32_t bnd_ip = 0;
uint16_t bnd_port = 0;
bool host_fd_is_listener = false;
std::deque<std::vector<uint8_t>> tx_queue;
~UdpFlow() override {
if (host_fd != -1 && !host_fd_is_listener) ::close(host_fd);
if (tcp_fd != -1) ::close(tcp_fd);
}
void handle_host_event(Context* ctx, int fd, uint32_t events) override;
bool is_stale(time_t now) const override;
void destroy(Context* ctx) override;
};
struct TcpFlow;
struct IcmpFlow : public Flow {
int host_fd = -1;
union {
IcmpFlowKey4 key4;
IcmpFlowKey6 key6;
};
bool is_redirected = false;
bool is_redirected;
union {
uint32_t orig_dest_ip4;
uint8_t orig_dest_ip6[IPV6_ADDR_LEN];
};
~IcmpFlow() override {
if (host_fd != -1) ::close(host_fd);
}
void handle_host_event(Context* ctx, int fd, uint32_t events) override;
bool is_stale(time_t now) const override;
void destroy(Context* ctx) override;
uint16_t orig_dest_port;
uint32_t redirect_ip4;
uint8_t redirect_ip6[IPV6_ADDR_LEN];
uint16_t redirect_port;
};
struct UdpFlow {
struct FlowHeader header;
/* UDP specific */
int tcp_fd;
bool use_socks5;
UdpSocks5State state;
uint32_t bnd_ip;
uint16_t bnd_port;
bool host_fd_is_listener;
struct {
uint8_t data[UDP_QUEUE_PACKET_MAX];
size_t len;
} c_tx_queue[UDP_QUEUE_MAX_PACKETS];
size_t c_tx_queue_head;
size_t c_tx_queue_tail;
size_t c_tx_queue_count;
};
struct TcpFlow {
struct FlowHeader header;
/* TCP specific */
TcpState tcp_state;
ProxyMode proxy_mode;
bool host_eof;
bool guest_eof;
bool fin_sent;
bool syn_acked;
bool fin_acked;
uint32_t seq_to_guest;
uint32_t ack_from_guest;
uint32_t seq_from_guest;
uint32_t ack_to_guest;
size_t tx_acked_offset;
size_t rx_sent_offset;
/* C-style buffers for migration */
uint8_t c_tcp_tx_buf[4096];
size_t c_tcp_tx_len;
uint8_t c_proxy_rx_buf[8192];
size_t c_proxy_rx_len;
uint8_t c_tcp_rx_buf[4096];
size_t c_tcp_rx_len;
bool epoll_out_registered;
bool epoll_in_disabled;
bool inbound;
};
struct IcmpFlow {
struct FlowHeader header;
};
struct Context {
int epoll_fd;
int tap_fd;
struct nsj_t* nsj;
@@ -159,24 +170,51 @@ struct Context {
uint8_t guest_ip6[IPV6_ADDR_LEN];
uint8_t host_ip6[IPV6_ADDR_LEN];
std::vector<nstun_rule_t> rules;
std::map<FlowKey4, std::unique_ptr<UdpFlow>> ipv4_udp_flows_by_key;
std::map<FlowKey4, std::unique_ptr<TcpFlow>> ipv4_tcp_flows_by_key;
std::map<IcmpFlowKey4, std::unique_ptr<IcmpFlow>> ipv4_icmp_flows_by_key;
/* Unified host mapping for all encapsulated flows */
std::map<int, Flow*> flows_by_fd; // Observer pointer
nstun_rule_t c_rules[NSTUN_MAX_RULES];
size_t c_rules_count;
/* IPv6 maps (Owning) */
std::map<FlowKey6, std::unique_ptr<UdpFlow>> ipv6_udp_flows_by_key;
std::map<FlowKey6, std::unique_ptr<TcpFlow>> ipv6_tcp_flows_by_key;
std::map<IcmpFlowKey6, std::unique_ptr<IcmpFlow>> ipv6_icmp_flows_by_key;
TcpFlow c_ipv4_tcp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv4_tcp_flows;
UdpFlow c_ipv4_udp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv4_udp_flows;
UdpFlow c_ipv6_udp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv6_udp_flows;
TcpFlow c_ipv6_tcp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv6_tcp_flows;
IcmpFlow c_ipv4_icmp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv4_icmp_flows;
IcmpFlow c_ipv6_icmp_flows[NSTUN_MAX_FLOWS];
size_t num_c_ipv6_icmp_flows;
struct {
int fd;
nstun_rule_t rule;
} c_host_listener_rules[NSTUN_MAX_RULES];
size_t num_c_host_listener_rules;
std::map<int, nstun_rule_t> host_listener_fd_to_rule;
/* Buffer for TUN frames, moved from TLS to avoid stack/TLS pressure */
uint8_t tun_buf[NSTUN_MTU + 4];
/* Buffers for recvmmsg, moved from TLS to avoid stack/TLS pressure */
struct mmsghdr recvmmsg_msgs[VLEN];
struct iovec recvmmsg_iovecs[VLEN];
uint8_t recvmmsg_bufs[VLEN][NSTUN_MTU];
struct sockaddr_storage recvmmsg_addrs[VLEN];
bool recvmmsg_initialized;
/* Buffer for SOCKS5 UDP control channel reads */
socks5_max_buf udp_socks5_buf;
/* Specific lookup tables for type safety */
TcpFlow* c_tcp_flows_by_fd[NSTUN_MAX_FDS] = {};
UdpFlow* c_udp_flows_by_fd[NSTUN_MAX_FDS] = {};
IcmpFlow* c_icmp_flows_by_fd[NSTUN_MAX_FDS] = {};
~Context();
};
void handle_host_events(Context* ctx, int fd, uint32_t events);
void host_callback(int fd, uint32_t events, void* data);
struct RuleResult {
nstun_action_t action;
uint32_t redirect_ip4;
@@ -185,7 +223,50 @@ struct RuleResult {
uint8_t redirect_ip6[IPV6_ADDR_LEN];
};
inline TcpFlow* get_tcp_flow_by_fd(const Context* ctx, int fd) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return nullptr;
}
return ctx->c_tcp_flows_by_fd[fd];
}
inline bool set_tcp_flow_by_fd(Context* ctx, int fd, TcpFlow* flow) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return false;
}
ctx->c_tcp_flows_by_fd[fd] = flow;
return true;
}
inline UdpFlow* get_udp_flow_by_fd(const Context* ctx, int fd) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return nullptr;
}
return ctx->c_udp_flows_by_fd[fd];
}
inline bool set_udp_flow_by_fd(Context* ctx, int fd, UdpFlow* flow) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return false;
}
ctx->c_udp_flows_by_fd[fd] = flow;
return true;
}
inline IcmpFlow* get_icmp_flow_by_fd(const Context* ctx, int fd) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return nullptr;
}
return ctx->c_icmp_flows_by_fd[fd];
}
inline bool set_icmp_flow_by_fd(Context* ctx, int fd, IcmpFlow* flow) {
if (fd < 0 || fd >= (int)NSTUN_MAX_FDS) {
return false;
}
ctx->c_icmp_flows_by_fd[fd] = flow;
return true;
}
} /* namespace nstun */

105
nstun/encap.cc
View File

@@ -5,6 +5,7 @@
#include <sys/socket.h>
#include "logs.h"
#include "macros.h"
#include "net_defs.h"
namespace nstun {
@@ -15,14 +16,17 @@ int send_socks5_greeting(int fd) {
.num_auth = 1,
.auth = {SOCKS5_AUTH_NONE},
};
if (send(fd, &greeting, sizeof(greeting), MSG_NOSIGNAL) != (ssize_t)sizeof(greeting)) {
if (TEMP_FAILURE_RETRY(send(fd, &greeting, sizeof(greeting), MSG_NOSIGNAL)) !=
(ssize_t)sizeof(greeting)) {
return -1;
}
return 0;
}
bool parse_socks5_auth_reply(std::span<const uint8_t> data) {
if (data.size() < 2) return false;
bool parse_socks5_auth_reply(const uint8_t* data, size_t len) {
if (len < 2) {
return false;
}
return data[0] == SOCKS5_VERSION && data[1] == SOCKS5_AUTH_NONE;
}
@@ -37,7 +41,8 @@ int send_socks5_connect(int fd, const uint8_t* addr, uint16_t port_nbo, bool is_
.dst_port = port_nbo, /* Already in network byte order */
};
memcpy(req.dst_ip6, addr, sizeof(req.dst_ip6));
if (send(fd, &req, sizeof(req), MSG_NOSIGNAL) != (ssize_t)sizeof(req)) {
if (TEMP_FAILURE_RETRY(send(fd, &req, sizeof(req), MSG_NOSIGNAL)) !=
(ssize_t)sizeof(req)) {
return -1;
}
return 0;
@@ -51,7 +56,8 @@ int send_socks5_connect(int fd, const uint8_t* addr, uint16_t port_nbo, bool is_
.dst_port = port_nbo,
};
memcpy(&req.dst_ip4, addr, 4);
if (send(fd, &req, sizeof(req), MSG_NOSIGNAL) != (ssize_t)sizeof(req)) {
if (TEMP_FAILURE_RETRY(send(fd, &req, sizeof(req), MSG_NOSIGNAL)) !=
(ssize_t)sizeof(req)) {
return -1;
}
return 0;
@@ -67,84 +73,113 @@ int send_socks5_udp_associate(int fd) {
.dst_ip4 = 0,
.dst_port = 0,
};
if (send(fd, &req, sizeof(req), MSG_NOSIGNAL) != (ssize_t)sizeof(req)) {
if (TEMP_FAILURE_RETRY(send(fd, &req, sizeof(req), MSG_NOSIGNAL)) != (ssize_t)sizeof(req)) {
return -1;
}
return 0;
}
bool parse_socks5_connect_reply(std::span<const uint8_t> data, Socks5Reply* out) {
if (data.size() < 4) return false;
bool parse_socks5_connect_reply(const uint8_t* data, size_t len, Socks5Reply* out) {
if (len < 4) {
return false;
}
/* data[0]=ver, data[1]=rep, data[2]=rsv, data[3]=atyp */
if (data[0] != SOCKS5_VERSION) return false;
if (data[1] != SOCKS5_REP_SUCCESS) return false;
if (data[0] != SOCKS5_VERSION) {
return false;
}
if (data[1] != SOCKS5_REP_SUCCESS) {
return false;
}
out->atyp = data[3];
if (out->atyp == SOCKS5_ATYP_IPV4) {
/* Full reply: 4-byte header + 4-byte IPv4 + 2-byte port */
if (data.size() < sizeof(socks5_req)) return false;
const auto* reply = reinterpret_cast<const socks5_req*>(data.data());
memcpy(&out->bind_ip4, &reply->dst_ip4, sizeof(reply->dst_ip4));
memcpy(&out->bind_port, &reply->dst_port, sizeof(reply->dst_port));
if (len < sizeof(socks5_req)) {
return false;
}
socks5_req reply;
memcpy(&reply, data, sizeof(reply));
memcpy(&out->bind_ip4, &reply.dst_ip4, sizeof(reply.dst_ip4));
memcpy(&out->bind_port, &reply.dst_port, sizeof(reply.dst_port));
} else if (out->atyp == SOCKS5_ATYP_IPV6) {
/* Full reply: 4-byte header + 16-byte IPv6 + 2-byte port */
if (data.size() < sizeof(socks5_req6)) return false;
const auto* reply = reinterpret_cast<const socks5_req6*>(data.data());
memcpy(&out->bind_port, &reply->dst_port, sizeof(reply->dst_port));
if (len < sizeof(socks5_req6)) {
return false;
}
socks5_req6 reply;
memcpy(&reply, data, sizeof(reply));
memcpy(&out->bind_port, &reply.dst_port, sizeof(reply.dst_port));
} else {
return false;
}
return true;
}
int send_http_connect(int fd, const uint8_t* addr, uint16_t port_nbo, bool is_ipv6) {
std::string addr_str =
is_ipv6 ? ip6_to_string(addr) : ip4_to_string(*(const uint32_t*)addr);
char addr_str[INET6_ADDRSTRLEN];
if (is_ipv6) {
inet_ntop(AF_INET6, addr, addr_str, sizeof(addr_str));
} else {
uint32_t ip4;
memcpy(&ip4, addr, 4);
inet_ntop(AF_INET, &ip4, addr_str, sizeof(addr_str));
}
uint16_t port = ntohs(port_nbo);
/* Max: "CONNECT [" + 39-char IPv6 + "]:65535 HTTP/1.1\r\nHost: [" + 39 + "]:65535\r\n\r\n"
*/
char buf[256];
char buf[160];
int n;
if (is_ipv6) {
n = snprintf(buf, sizeof(buf), "CONNECT [%s]:%u HTTP/1.1\r\nHost: [%s]:%u\r\n\r\n",
addr_str.c_str(), port, addr_str.c_str(), port);
addr_str, port, addr_str, port);
} else {
n = snprintf(buf, sizeof(buf), "CONNECT %s:%u HTTP/1.1\r\nHost: %s:%u\r\n\r\n",
addr_str.c_str(), port, addr_str.c_str(), port);
addr_str, port, addr_str, port);
}
if (n <= 0 || n >= (int)sizeof(buf)) return -1;
if (send(fd, buf, n, MSG_NOSIGNAL) != (ssize_t)n) return -1;
if (n <= 0 || n >= (int)sizeof(buf)) {
return -1;
}
if (TEMP_FAILURE_RETRY(send(fd, buf, n, MSG_NOSIGNAL)) != (ssize_t)n) {
return -1;
}
return 0;
}
size_t find_end_of_headers(const std::vector<uint8_t>& buf) {
for (size_t i = 0; i + 3 < buf.size(); ++i) {
if (buf[i] == '\r' && buf[i + 1] == '\n' && buf[i + 2] == '\r' &&
buf[i + 3] == '\n') {
size_t find_end_of_headers(const uint8_t* data, size_t len) {
for (size_t i = 0; i + 3 < len; ++i) {
if (data[i] == '\r' && data[i + 1] == '\n' && data[i + 2] == '\r' &&
data[i + 3] == '\n') {
return i + 4;
}
}
return 0;
}
bool parse_http_connect_reply(const std::vector<uint8_t>& buf) {
bool parse_http_connect_reply(const uint8_t* data, size_t len) {
/* Minimum valid response: "HTTP/1.x 2xx" = 12 chars.
* We only check the version prefix and status class (2xx). */
static constexpr std::string_view HTTP_VERSION_PREFIX = "HTTP/1.";
const char* HTTP_VERSION_PREFIX = "HTTP/1.";
/* Offset of the first status digit in "HTTP/1.x NNN": H(0)T(1)T(2)P(3)/(4)1(5).(6)x(7)
* (8)N(9) */
static constexpr size_t HTTP_STATUS_DIGIT_OFFSET = 9;
static constexpr size_t HTTP_RESPONSE_MIN_LEN =
constexpr size_t HTTP_STATUS_DIGIT_OFFSET = 9;
constexpr size_t HTTP_RESPONSE_MIN_LEN =
HTTP_STATUS_DIGIT_OFFSET + 3; /* digit + two more chars of status */
if (buf.size() < HTTP_RESPONSE_MIN_LEN) return false;
if (memcmp(buf.data(), HTTP_VERSION_PREFIX.data(), HTTP_VERSION_PREFIX.size()) != 0)
if (len < HTTP_RESPONSE_MIN_LEN) {
return false;
}
if (memcmp(data, HTTP_VERSION_PREFIX, 7) != 0) {
return false;
}
/* Accept any 2xx status code */
if (buf[HTTP_STATUS_DIGIT_OFFSET] != '2') return false;
if (data[HTTP_STATUS_DIGIT_OFFSET] != '2') {
return false;
}
return true;
}

13
nstun/encap.h
View File

@@ -3,9 +3,6 @@
#include <stdint.h>
#include <span>
#include <vector>
#include "net_defs.h"
namespace nstun {
@@ -93,15 +90,17 @@ struct Socks5Reply {
};
int send_socks5_greeting(int fd);
bool parse_socks5_auth_reply(std::span<const uint8_t> data);
bool parse_socks5_auth_reply(const uint8_t* data, size_t len);
int send_socks5_connect(int fd, const uint8_t* addr, uint16_t port_nbo, bool is_ipv6);
int send_socks5_udp_associate(int fd);
bool parse_socks5_connect_reply(std::span<const uint8_t> data, Socks5Reply* out);
bool parse_socks5_connect_reply(const uint8_t* data, size_t len, Socks5Reply* out);
/* HTTP CONNECT helpers */
int send_http_connect(int fd, const uint8_t* addr, uint16_t port_nbo, bool is_ipv6);
size_t find_end_of_headers(const std::vector<uint8_t>& buf);
bool parse_http_connect_reply(const std::vector<uint8_t>& buf);
size_t find_end_of_headers(const uint8_t* data, size_t len);
bool parse_http_connect_reply(const uint8_t* data, size_t len);
} /* namespace nstun */

690
nstun/icmp.cc
View File

@@ -4,6 +4,7 @@
#include <netinet/icmp6.h>
#include <netinet/in.h>
#include <netinet/ip_icmp.h>
#include <stdio.h>
#include <string.h>
#include <sys/epoll.h>
#include <sys/socket.h>
@@ -17,94 +18,166 @@
namespace nstun {
static IcmpFlow* find_ipv4_icmp_flow(Context* ctx, const IcmpFlowKey4& key4) {
size_t active_seen = 0;
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
IcmpFlow& flow = ctx->c_ipv4_icmp_flows[i];
if (flow.header.active) {
if (memcmp(&flow.header.icmp_key4, &key4, sizeof(key4)) == 0) {
flow.header.last_active = time(nullptr);
return &flow;
}
active_seen++;
if (active_seen >= ctx->num_c_ipv4_icmp_flows) {
break;
}
}
}
return nullptr;
}
static IcmpFlow* find_ipv6_icmp_flow(Context* ctx, const IcmpFlowKey6& key6) {
size_t active_seen = 0;
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
IcmpFlow& flow = ctx->c_ipv6_icmp_flows[i];
if (flow.header.active) {
if (memcmp(&flow.header.icmp_key6, &key6, sizeof(key6)) == 0) {
flow.header.last_active = time(nullptr);
return &flow;
}
active_seen++;
if (active_seen >= ctx->num_c_ipv6_icmp_flows) {
break;
}
}
}
return nullptr;
}
static IcmpFlow* alloc_icmp_flow(IcmpFlow* flows, size_t max_flows) {
for (size_t i = 0; i < max_flows; ++i) {
if (!flows[i].header.active) {
return &flows[i];
}
}
return nullptr;
}
static IcmpFlow* alloc_ipv4_icmp_flow(Context* ctx) {
return alloc_icmp_flow(ctx->c_ipv4_icmp_flows, NSTUN_MAX_FLOWS);
}
static IcmpFlow* alloc_ipv6_icmp_flow(Context* ctx) {
return alloc_icmp_flow(ctx->c_ipv6_icmp_flows, NSTUN_MAX_FLOWS);
}
static void init_icmp_flow_zero(IcmpFlow* flow) {
memset(flow, 0, sizeof(IcmpFlow));
flow->header.type = FlowType::ICMP;
flow->header.host_fd = -1;
}
/* Maximum frame size for an ICMP error packet we will generate.
* RFC 792 / RFC 4443 require including the original IP header + 8 bytes
* of the triggering datagram. We cap total frame size conservatively. */
static constexpr size_t ICMP_ERROR_MAX_FRAME = 128;
static void icmp_destroy_flow(Context* ctx, IcmpFlow* flow) {
if (flow->host_fd != -1) {
epoll_ctl(ctx->epoll_fd, EPOLL_CTL_DEL, flow->host_fd, nullptr);
ctx->flows_by_fd.erase(flow->host_fd);
/* close() is handled by ~IcmpFlow via unique_ptr destruction below */
if (!flow->header.active) {
return;
}
if (flow->is_ipv6) {
ctx->ipv6_icmp_flows_by_key.erase(flow->key6);
if (flow->header.host_fd != -1) {
monitor::removeFd(flow->header.host_fd);
close(flow->header.host_fd);
if (flow->header.host_fd >= 0 &&
static_cast<size_t>(flow->header.host_fd) < nstun::NSTUN_MAX_FDS) {
ctx->c_icmp_flows_by_fd[flow->header.host_fd] = nullptr;
}
flow->header.host_fd = -1;
}
flow->header.active = false;
if (flow->header.is_ipv6) {
ctx->num_c_ipv6_icmp_flows--;
} else {
ctx->ipv4_icmp_flows_by_key.erase(flow->key4);
ctx->num_c_ipv4_icmp_flows--;
}
/* unique_ptr in the owning map runs ~IcmpFlow() which closes host_fd */
}
static void icmp_send_packet4(Context* ctx, uint32_t saddr, uint32_t daddr, uint8_t type,
uint8_t code, uint16_t id, uint16_t seq, const uint8_t* data, size_t len) {
static bool icmp_send_packet4(Context* ctx, uint32_t saddr, uint32_t daddr, uint8_t type,
uint8_t code, uint16_t id, uint16_t seq, const void* data, size_t len) {
if (len > NSTUN_MTU) {
LOG_W("icmp_send_packet4: data length too large");
return;
return false;
}
size_t frame_len = sizeof(ip4_hdr) + sizeof(icmp4_hdr) + len;
static thread_local uint8_t header_buf[sizeof(ip4_hdr) + sizeof(icmp4_hdr)];
uint8_t header_buf[sizeof(ip4_hdr) + sizeof(icmp4_hdr)];
memset(header_buf, 0, sizeof(header_buf));
ip4_hdr* r_ip = reinterpret_cast<ip4_hdr*>(header_buf);
icmp4_hdr* r_icmp = reinterpret_cast<icmp4_hdr*>(header_buf + sizeof(ip4_hdr));
ip4_hdr ip = {};
icmp4_hdr icmp = {};
/* IPv4 */
ip4_set_ihl_version(r_ip, 4, sizeof(ip4_hdr) / 4);
r_ip->tos = 0;
r_ip->tot_len = htons(frame_len);
r_ip->id = 0;
r_ip->frag_off = 0;
r_ip->ttl = 64;
r_ip->protocol = IPPROTO_ICMP;
r_ip->saddr = saddr;
r_ip->daddr = daddr;
r_ip->check = 0;
r_ip->check = compute_checksum(r_ip, sizeof(ip4_hdr));
ip4_set_ihl_version(&ip, 4, sizeof(ip4_hdr) / 4);
ip.tos = 0;
ip.tot_len = htons(frame_len);
ip.id = 0;
ip.frag_off = 0;
ip.ttl = 64;
ip.protocol = IPPROTO_ICMP;
ip.saddr = saddr;
ip.daddr = daddr;
ip.check = 0;
ip.check = compute_checksum(&ip, sizeof(ip4_hdr));
/* ICMP */
r_icmp->type = type;
r_icmp->code = code;
r_icmp->id = id;
r_icmp->seq = seq;
r_icmp->check = 0;
icmp.type = type;
icmp.code = code;
icmp.id = id;
icmp.seq = seq;
icmp.check = 0;
uint32_t sum = compute_checksum_part(r_icmp, sizeof(icmp4_hdr), 0);
uint32_t sum = compute_checksum_part(&icmp, sizeof(icmp4_hdr), 0);
if (data && len > 0) {
sum = compute_checksum_part(data, len, sum);
}
r_icmp->check = finalize_checksum(sum);
icmp.check = finalize_checksum(sum);
send_to_guest_v(ctx, header_buf, sizeof(header_buf), data, len);
memcpy(header_buf, &ip, sizeof(ip));
memcpy(header_buf + sizeof(ip), &icmp, sizeof(icmp));
return send_to_guest_v(
ctx, header_buf, sizeof(header_buf), static_cast<const uint8_t*>(data), len);
}
static void icmp_send_packet6(Context* ctx, const uint8_t* saddr, const uint8_t* daddr,
uint8_t type, uint8_t code, uint16_t id, uint16_t seq, const uint8_t* data, size_t len) {
static bool icmp_send_packet6(Context* ctx, const uint8_t* saddr, const uint8_t* daddr,
uint8_t type, uint8_t code, uint16_t id, uint16_t seq, const void* data, size_t len) {
if (len > NSTUN_MTU) {
LOG_W("icmp_send_packet6: data length too large");
return;
return false;
}
static thread_local uint8_t header_buf[sizeof(ip6_hdr) + sizeof(icmp6_hdr)];
uint8_t header_buf[sizeof(ip6_hdr) + sizeof(icmp6_hdr)];
memset(header_buf, 0, sizeof(header_buf));
ip6_hdr* r_ip6 = reinterpret_cast<ip6_hdr*>(header_buf);
icmp6_hdr* r_icmp6 = reinterpret_cast<icmp6_hdr*>(header_buf + sizeof(ip6_hdr));
ip6_hdr ip6 = {};
icmp6_hdr icmp6 = {};
/* IPv6 */
r_ip6->vtf = htonl(0x60000000); /* Version 6 */
r_ip6->payload_len = htons(sizeof(icmp6_hdr) + len);
r_ip6->next_header = IPPROTO_ICMPV6;
r_ip6->hop_limit = 64;
memcpy(r_ip6->saddr, saddr, sizeof(r_ip6->saddr));
memcpy(r_ip6->daddr, daddr, sizeof(r_ip6->daddr));
ip6.vtf = htonl(0x60000000); /* Version 6 */
ip6.payload_len = htons(sizeof(icmp6_hdr) + len);
ip6.next_header = IPPROTO_ICMPV6;
ip6.hop_limit = 64;
memcpy(ip6.saddr, saddr, sizeof(ip6.saddr));
memcpy(ip6.daddr, daddr, sizeof(ip6.daddr));
/* ICMPv6 */
r_icmp6->type = type;
r_icmp6->code = code;
r_icmp6->id = id;
r_icmp6->seq = seq;
r_icmp6->check = 0;
icmp6.type = type;
icmp6.code = code;
icmp6.id = id;
icmp6.seq = seq;
icmp6.check = 0;
/* 40-byte IPv6 pseudo header */
pseudo_hdr6 phdr = {};
@@ -114,13 +187,17 @@ static void icmp_send_packet6(Context* ctx, const uint8_t* saddr, const uint8_t*
memcpy(phdr.daddr, daddr, sizeof(phdr.daddr));
uint32_t sum = compute_checksum_part(&phdr, sizeof(phdr), 0);
sum = compute_checksum_part(r_icmp6, sizeof(icmp6_hdr), sum);
sum = compute_checksum_part(&icmp6, sizeof(icmp6_hdr), sum);
if (data && len > 0) {
sum = compute_checksum_part(data, len, sum);
}
r_icmp6->check = finalize_checksum(sum);
icmp6.check = finalize_checksum(sum);
send_to_guest_v(ctx, header_buf, sizeof(header_buf), data, len);
memcpy(header_buf, &ip6, sizeof(ip6));
memcpy(header_buf + sizeof(ip6), &icmp6, sizeof(icmp6));
return send_to_guest_v(
ctx, header_buf, sizeof(header_buf), static_cast<const uint8_t*>(data), len);
}
void send_icmp4_error(
@@ -129,12 +206,18 @@ void send_icmp4_error(
/* RFC 792: include IP header + first 8 bytes of original datagram.
* Clamp to actual available data to prevent OOB read. */
size_t icmp_data_len = req_ihl + 8;
if (icmp_data_len > tot_len) icmp_data_len = tot_len;
if (icmp_data_len > tot_len) {
icmp_data_len = tot_len;
}
if (sizeof(ip4_hdr) + sizeof(icmp4_hdr) + icmp_data_len > ICMP_ERROR_MAX_FRAME) return;
if (sizeof(ip4_hdr) + sizeof(icmp4_hdr) + icmp_data_len > ICMP_ERROR_MAX_FRAME) {
return;
}
icmp_send_packet4(ctx, req_ip->daddr, req_ip->saddr, type, code, 0, 0,
reinterpret_cast<const uint8_t*>(req_ip), icmp_data_len);
if (!icmp_send_packet4(
ctx, req_ip->daddr, req_ip->saddr, type, code, 0, 0, req_ip, icmp_data_len)) {
LOG_W("send_icmp4_error: failed to send ICMP error to guest");
}
}
void send_icmp6_error(
@@ -142,29 +225,122 @@ void send_icmp6_error(
/* IPv6 header + first 8 bytes of original datagram.
* Clamp to actual available data to prevent OOB read. */
size_t icmp_data_len = sizeof(ip6_hdr) + 8;
if (icmp_data_len > tot_len) icmp_data_len = tot_len;
if (icmp_data_len > tot_len) {
icmp_data_len = tot_len;
}
if (sizeof(ip6_hdr) + sizeof(icmp6_hdr) + icmp_data_len > ICMP_ERROR_MAX_FRAME) return;
icmp_send_packet6(ctx, req_ip->daddr, req_ip->saddr, type, code, 0, 0,
reinterpret_cast<const uint8_t*>(req_ip), icmp_data_len);
}
void handle_icmp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payload) {
if (payload.size() < sizeof(icmp6_hdr)) {
if (sizeof(ip6_hdr) + sizeof(icmp6_hdr) + icmp_data_len > ICMP_ERROR_MAX_FRAME) {
return;
}
const icmp6_hdr* icmp = reinterpret_cast<const icmp6_hdr*>(payload.data());
if (!icmp_send_packet6(
ctx, req_ip->daddr, req_ip->saddr, type, code, 0, 0, req_ip, icmp_data_len)) {
LOG_W("send_icmp6_error: failed to send ICMPv6 error to guest");
}
}
static void proxy_icmp6(Context* ctx, const ip6_hdr* ip, const icmp6_hdr* icmp,
const uint8_t* payload, size_t len, const RuleResult& rule) {
/* Attempt to proxy ICMP using unprivileged socket */
IcmpFlowKey6 key6 = {};
memcpy(key6.saddr6, ip->saddr, sizeof(key6.saddr6));
memcpy(key6.daddr6, ip->daddr, sizeof(key6.daddr6));
key6.id = icmp->id;
IcmpFlow* flow = find_ipv6_icmp_flow(ctx, key6);
if (!flow) {
if (ctx->num_c_ipv6_icmp_flows >= NSTUN_MAX_FLOWS) {
LOG_W("Maximum number of IPv6 ICMP flows reached, dropping");
return;
}
flow = alloc_ipv6_icmp_flow(ctx);
if (!flow) {
LOG_E("Failed to allocate Flow (table full)");
return;
}
int fd =
socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC, IPPROTO_ICMPV6);
if (fd == -1) {
PLOG_W("socket(AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6) failed.");
send_icmp6_error(ctx, ip, sizeof(ip6_hdr) + len, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_ADDR);
return;
}
if (fd >= (int)nstun::NSTUN_MAX_FDS) {
LOG_E("FD limit reached (fd=%d)", fd);
close(fd);
return;
}
if (!monitor::addFd(fd, EPOLLIN, host_callback, ctx)) {
PLOG_E("monitor::addFd() for ICMPv6 failed");
close(fd);
return;
}
init_icmp_flow_zero(flow);
flow->header.active = true;
flow->header.host_fd = fd;
flow->header.is_ipv6 = true;
flow->header.icmp_key6 = key6;
flow->header.last_active = time(nullptr);
flow->header.is_redirected = rule.has_redirect_ip6;
memcpy(flow->header.orig_dest_ip6, ip->daddr, sizeof(flow->header.orig_dest_ip6));
if (rule.has_redirect_ip6) {
memcpy(flow->header.redirect_ip6, rule.redirect_ip6,
sizeof(flow->header.redirect_ip6));
}
ctx->num_c_ipv6_icmp_flows++;
if (fd >= 0 && static_cast<size_t>(fd) < nstun::NSTUN_MAX_FDS) {
ctx->c_icmp_flows_by_fd[fd] = flow;
}
if (flow->header.is_redirected) {
LOG_D("Created IPv6 ICMP flow for ID %u (fd=%d) [redirected]",
ntohs(key6.id), fd);
} else {
LOG_D("Created IPv6 ICMP flow for ID %u (fd=%d)", ntohs(key6.id), fd);
}
}
struct sockaddr_in6 dest_addr = INIT_SOCKADDR_IN6(AF_INET6);
if (flow->header.is_redirected) {
memcpy(
&dest_addr.sin6_addr, flow->header.redirect_ip6, sizeof(dest_addr.sin6_addr));
} else {
/* ip.cc already rejected loopback/v4-mapped destinations */
memcpy(
&dest_addr.sin6_addr, flow->header.orig_dest_ip6, sizeof(dest_addr.sin6_addr));
}
ssize_t sent = sendto(flow->header.host_fd, payload, len, MSG_NOSIGNAL,
reinterpret_cast<const sockaddr*>(&dest_addr), sizeof(dest_addr));
if (sent == -1) {
PLOG_E("sendto(fd=%d) ICMPv6 failed", flow->header.host_fd);
}
}
void handle_icmp6(Context* ctx, const ip6_hdr* ip, const uint8_t* payload, size_t len) {
if (len < sizeof(icmp6_hdr)) {
return;
}
icmp6_hdr icmp_storage;
memcpy(&icmp_storage, payload, sizeof(icmp_storage));
const icmp6_hdr* icmp = &icmp_storage;
/* Validate ICMPv6 checksum (mandatory per RFC 4443) */
pseudo_hdr6 phdr = {};
phdr.len = htonl(payload.size());
phdr.len = htonl(len);
phdr.next_header = IPPROTO_ICMPV6;
memcpy(phdr.saddr, ip->saddr, sizeof(phdr.saddr));
memcpy(phdr.daddr, ip->daddr, sizeof(phdr.daddr));
uint32_t csum = compute_checksum_part(&phdr, sizeof(phdr), 0);
csum = compute_checksum_part(payload.data(), payload.size(), csum);
csum = compute_checksum_part(payload, len, csum);
if (finalize_checksum(csum) != 0) {
LOG_D("ICMPv6 checksum invalid, dropping");
return;
@@ -180,8 +356,7 @@ void handle_icmp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payl
return;
} else if (rule.action == NSTUN_ACTION_REJECT) {
LOG_D("ICMPv6 rejected by policy");
send_icmp6_error(ctx, ip, sizeof(ip6_hdr) + payload.size(),
ICMP6_DST_UNREACH,
send_icmp6_error(ctx, ip, sizeof(ip6_hdr) + len, ICMP6_DST_UNREACH,
ICMP6_DST_UNREACH_NOPORT); /* Dest unreachable, port unreachable */
return;
}
@@ -189,112 +364,120 @@ void handle_icmp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payl
if (memcmp(ip->daddr, ctx->host_ip6, IPV6_ADDR_LEN) == 0 &&
rule.action != NSTUN_ACTION_REDIRECT) {
/* Construct reply (Type 129, Code 0) */
const uint8_t* icmp_payload = payload.data() + sizeof(icmp6_hdr);
size_t icmp_payload_len = payload.size() - sizeof(icmp6_hdr);
icmp_send_packet6(ctx, ip->daddr, ip->saddr, ICMP6_ECHO_REPLY, 0, icmp->id,
icmp->seq, icmp_payload, icmp_payload_len);
const uint8_t* icmp_payload = payload + sizeof(icmp6_hdr);
size_t icmp_payload_len = len - sizeof(icmp6_hdr);
if (!icmp_send_packet6(ctx, ip->daddr, ip->saddr, ICMP6_ECHO_REPLY, 0,
icmp->id, icmp->seq, icmp_payload, icmp_payload_len)) {
LOG_W("handle_icmp6: failed to send Echo Reply");
}
} else {
/* Attempt to proxy ICMP using unprivileged socket */
IcmpFlowKey6 key6 = {};
memcpy(key6.saddr6, ip->saddr, sizeof(key6.saddr6));
memcpy(key6.daddr6, ip->daddr, sizeof(key6.daddr6));
key6.id = icmp->id;
IcmpFlow* flow = nullptr;
auto it = ctx->ipv6_icmp_flows_by_key.find(key6);
if (it != ctx->ipv6_icmp_flows_by_key.end()) {
flow = it->second.get();
flow->last_active = time(NULL);
} else {
if (ctx->ipv6_icmp_flows_by_key.size() >= NSTUN_MAX_FLOWS) {
LOG_W("Maximum number of IPv6 ICMP flows (%zu) reached, "
"dropping",
NSTUN_MAX_FLOWS);
return;
}
int fd = socket(AF_INET6, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC,
IPPROTO_ICMPV6);
if (fd == -1) {
PLOG_W("socket(AF_INET6, SOCK_DGRAM, IPPROTO_ICMPV6) "
"failed.");
return;
}
bool success = false;
defer {
if (!success) {
close(fd);
}
};
struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = fd}};
if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) {
PLOG_E("epoll_ctl(EPOLL_CTL_ADD) for ICMPv6 failed");
return;
}
success = true;
auto flow_ptr = std::make_unique<IcmpFlow>();
flow = flow_ptr.get();
flow->host_fd = fd;
flow->is_ipv6 = true;
flow->key6 = key6;
flow->last_active = time(NULL);
flow->is_redirected = rule.has_redirect_ip6;
memcpy(flow->orig_dest_ip6, ip->daddr, sizeof(flow->orig_dest_ip6));
ctx->ipv6_icmp_flows_by_key[key6] = std::move(flow_ptr);
ctx->flows_by_fd[fd] = flow;
if (flow->is_redirected) {
LOG_D(
"Created IPv6 ICMP flow for ID %u (fd=%d) [redirected]",
ntohs(key6.id), fd);
} else {
LOG_D("Created IPv6 ICMP flow for ID %u (fd=%d)",
ntohs(key6.id), fd);
}
}
struct sockaddr_in6 dest_addr = INIT_SOCKADDR_IN6(AF_INET6);
if (rule.has_redirect_ip6) {
memcpy(&dest_addr.sin6_addr, rule.redirect_ip6,
sizeof(dest_addr.sin6_addr));
} else {
/* ip.cc already rejected loopback/v4-mapped destinations */
memcpy(
&dest_addr.sin6_addr, ip->daddr, sizeof(dest_addr.sin6_addr));
}
ssize_t sent = sendto(flow->host_fd, payload.data(), payload.size(),
MSG_NOSIGNAL, (struct sockaddr*)&dest_addr, sizeof(dest_addr));
if (sent == -1) {
PLOG_E("sendto(fd=%d) ICMPv6 failed", flow->host_fd);
}
proxy_icmp6(ctx, ip, icmp, payload, len, rule);
}
}
}
void handle_icmp4(Context* ctx, const ip4_hdr* ip, std::span<const uint8_t> payload) {
if (payload.size() < sizeof(icmp4_hdr)) {
static void proxy_icmp4(Context* ctx, const ip4_hdr* ip, const icmp4_hdr* icmp,
const uint8_t* payload, size_t len, const RuleResult& rule) {
/* Attempt to proxy ICMP using unprivileged socket */
IcmpFlowKey4 key4 = {ip->saddr, ip->daddr, icmp->id};
LOG_D("handle_icmp4: looking up flow");
IcmpFlow* flow = find_ipv4_icmp_flow(ctx, key4);
LOG_D("handle_icmp4: flow lookup done, flow=%p", flow);
if (!flow) {
if (ctx->num_c_ipv4_icmp_flows >= NSTUN_MAX_FLOWS) {
LOG_W("Maximum number of ICMP flows reached, dropping");
return;
}
flow = alloc_ipv4_icmp_flow(ctx);
if (!flow) {
LOG_E("Failed to allocate Flow (table full)");
return;
}
int fd = socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC, IPPROTO_ICMP);
if (fd == -1) {
PLOG_W("socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP) failed. "
"You may need: sysctl -w net.ipv4.ping_group_range='0 2147483647'");
send_icmp4_error(ctx, ip, ntohs(ip->tot_len), ICMP_DEST_UNREACH,
ICMP_HOST_UNREACH); /* host unreachable */
return;
}
if (fd >= (int)nstun::NSTUN_MAX_FDS) {
LOG_E("FD limit reached (fd=%d)", fd);
close(fd);
return;
}
if (!monitor::addFd(fd, EPOLLIN, host_callback, ctx)) {
PLOG_E("monitor::addFd() for ICMP failed");
close(fd);
return;
}
/* Initialize flow */
init_icmp_flow_zero(flow);
flow->header.active = true;
flow->header.host_fd = fd;
flow->header.is_ipv6 = false;
flow->header.icmp_key4 = key4;
flow->header.last_active = time(nullptr);
flow->header.is_redirected = (rule.redirect_ip4 != 0);
flow->header.orig_dest_ip4 = ip->daddr;
flow->header.redirect_ip4 = rule.redirect_ip4;
ctx->num_c_ipv4_icmp_flows++;
if (fd >= 0 && static_cast<size_t>(fd) < nstun::NSTUN_MAX_FDS) {
ctx->c_icmp_flows_by_fd[fd] = flow;
}
if (flow->header.is_redirected) {
LOG_D("Created ICMP flow for ID %u (fd=%d) [redirected to %s]",
ntohs(key4.id), fd, ip4_to_string(rule.redirect_ip4).c_str());
} else {
LOG_D("Created ICMP flow for ID %u (fd=%d)", ntohs(key4.id), fd);
}
}
struct sockaddr_in dest_addr = INIT_SOCKADDR_IN(AF_INET);
dest_addr.sin_addr.s_addr =
flow->header.is_redirected ? flow->header.redirect_ip4 : flow->header.orig_dest_ip4;
ssize_t sent = sendto(flow->header.host_fd, payload, len, MSG_NOSIGNAL,
reinterpret_cast<const sockaddr*>(&dest_addr), sizeof(dest_addr));
if (sent == -1) {
PLOG_E("sendto(fd=%d) ICMP failed", flow->header.host_fd);
}
}
void handle_icmp4(Context* ctx, const ip4_hdr* ip, const uint8_t* payload, size_t len) {
LOG_D("handle_icmp4: started, payload size=%zu", len);
if (len < sizeof(icmp4_hdr)) {
LOG_D("handle_icmp4: payload too small");
return;
}
const icmp4_hdr* icmp = reinterpret_cast<const icmp4_hdr*>(payload.data());
icmp4_hdr icmp_storage;
memcpy(&icmp_storage, payload, sizeof(icmp_storage));
const icmp4_hdr* icmp = &icmp_storage;
/* Validate ICMP checksum */
if (compute_checksum(payload.data(), payload.size()) != 0) {
LOG_D("handle_icmp4: validating checksum");
if (compute_checksum(payload, len) != 0) {
LOG_D("Invalid ICMP checksum, dropping");
return;
}
/* We only handle Echo Request (Type 8, Code 0) */
LOG_D("handle_icmp4: type=%u, code=%u", icmp->type, icmp->code);
if (icmp->type == ICMP_ECHO && icmp->code == 0) {
LOG_D("handle_icmp4: evaluating rules");
RuleResult rule = evaluate_rules4(
ctx, NSTUN_DIR_GUEST_TO_HOST, NSTUN_PROTO_ICMP, ip->saddr, ip->daddr, 0, 0);
LOG_D("handle_icmp4: rule action=%d", rule.action);
if (rule.action == NSTUN_ACTION_DROP) {
LOG_D("ICMP dropped by policy");
return;
@@ -305,120 +488,34 @@ void handle_icmp4(Context* ctx, const ip4_hdr* ip, std::span<const uint8_t> payl
return;
}
LOG_D("handle_icmp4: checking dest IP");
if (ip->daddr == ctx->host_ip4 && rule.action != NSTUN_ACTION_REDIRECT) {
LOG_D("handle_icmp4: sending echo reply");
/* Construct reply (Type 0, Code 0) */
const uint8_t* icmp_payload = payload.data() + sizeof(icmp4_hdr);
size_t icmp_payload_len = payload.size() - sizeof(icmp4_hdr);
icmp_send_packet4(ctx, ip->daddr, ip->saddr, ICMP_ECHOREPLY, 0, icmp->id,
icmp->seq, icmp_payload, icmp_payload_len);
const uint8_t* icmp_payload = payload + sizeof(icmp4_hdr);
size_t icmp_payload_len = len - sizeof(icmp4_hdr);
if (!icmp_send_packet4(ctx, ip->daddr, ip->saddr, ICMP_ECHOREPLY, 0,
icmp->id, icmp->seq, icmp_payload, icmp_payload_len)) {
LOG_W("handle_icmp4: failed to send Echo Reply");
}
} else {
/* Attempt to proxy ICMP using unprivileged socket */
IcmpFlowKey4 key4 = {ip->saddr, ip->daddr, icmp->id};
IcmpFlow* flow = nullptr;
auto it = ctx->ipv4_icmp_flows_by_key.find(key4);
if (it != ctx->ipv4_icmp_flows_by_key.end()) {
flow = it->second.get();
flow->last_active = time(NULL);
} else {
if (ctx->ipv4_icmp_flows_by_key.size() >= NSTUN_MAX_FLOWS) {
LOG_W(
"Maximum number of ICMP flows (%zu) reached, dropping",
NSTUN_MAX_FLOWS);
return;
}
int fd = socket(AF_INET, SOCK_DGRAM | SOCK_NONBLOCK | SOCK_CLOEXEC,
IPPROTO_ICMP);
if (fd == -1) {
PLOG_W(
"socket(AF_INET, SOCK_DGRAM, IPPROTO_ICMP) "
"failed. "
"You may need: sysctl -w net.ipv4.ping_group_range='0 "
"2147483647'");
send_icmp4_error(ctx, ip, ntohs(ip->tot_len),
ICMP_DEST_UNREACH,
ICMP_HOST_UNREACH); /* host unreachable */
return;
}
bool success = false;
defer {
if (!success) {
close(fd);
}
};
struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = fd}};
if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) {
PLOG_E("epoll_ctl(EPOLL_CTL_ADD) for ICMP failed");
return;
}
success = true;
auto flow_ptr = std::make_unique<IcmpFlow>();
flow = flow_ptr.get();
flow->host_fd = fd;
flow->is_ipv6 = false;
flow->key4 = key4;
flow->last_active = time(NULL);
flow->is_redirected = (rule.redirect_ip4 != 0);
flow->orig_dest_ip4 = ip->daddr;
ctx->ipv4_icmp_flows_by_key[key4] = std::move(flow_ptr);
ctx->flows_by_fd[fd] = flow;
if (flow->is_redirected) {
LOG_D("Created ICMP flow for ID %u (fd=%d) [redirected to "
"%s]",
ntohs(key4.id), fd,
ip4_to_string(rule.redirect_ip4).c_str());
} else {
LOG_D("Created ICMP flow for ID %u (fd=%d)", ntohs(key4.id),
fd);
}
}
/* ip.cc already rejected loopback destinations */
struct sockaddr_in dest_addr = INIT_SOCKADDR_IN(AF_INET);
dest_addr.sin_addr.s_addr =
(rule.redirect_ip4 != 0) ? rule.redirect_ip4 : ip->daddr;
ssize_t sent = sendto(flow->host_fd, payload.data(), payload.size(),
MSG_NOSIGNAL, (struct sockaddr*)&dest_addr, sizeof(dest_addr));
if (sent == -1) {
PLOG_E("sendto(fd=%d) ICMP failed", flow->host_fd);
}
LOG_D("handle_icmp4: proxying ICMP");
proxy_icmp4(ctx, ip, icmp, payload, len, rule);
}
}
}
static void handle_host_icmp(Context* ctx, IcmpFlow* flow) {
int fd = flow->host_fd;
flow->last_active = time(NULL);
constexpr int VLEN = 64;
struct mmsghdr msgs[VLEN];
struct iovec iovecs[VLEN];
static thread_local std::unique_ptr<uint8_t[][NSTUN_MTU]> bufs_ptr;
if (!bufs_ptr) {
bufs_ptr = std::make_unique<uint8_t[][NSTUN_MTU]>(VLEN);
}
uint8_t (*bufs)[NSTUN_MTU] = bufs_ptr.get();
static thread_local struct sockaddr_storage src_addrs[VLEN];
int fd = flow->header.host_fd;
flow->header.last_active = time(nullptr);
for (int i = 0; i < VLEN; ++i) {
iovecs[i].iov_base = bufs[i];
iovecs[i].iov_len = sizeof(bufs[i]);
msgs[i].msg_hdr.msg_iov = &iovecs[i];
msgs[i].msg_hdr.msg_iovlen = 1;
msgs[i].msg_hdr.msg_name = &src_addrs[i];
msgs[i].msg_hdr.msg_namelen = sizeof(src_addrs[i]);
msgs[i].msg_hdr.msg_control = nullptr;
msgs[i].msg_hdr.msg_controllen = 0;
ctx->recvmmsg_msgs[i].msg_hdr.msg_namelen = sizeof(ctx->recvmmsg_addrs[i]);
ctx->recvmmsg_msgs[i].msg_hdr.msg_controllen = 0;
}
int retval = recvmmsg(fd, msgs, VLEN, MSG_DONTWAIT, nullptr);
int retval =
TEMP_FAILURE_RETRY(recvmmsg(fd, ctx->recvmmsg_msgs, VLEN, MSG_DONTWAIT, nullptr));
if (retval == -1) {
if (errno == EAGAIN || errno == EWOULDBLOCK) {
return;
@@ -427,68 +524,81 @@ static void handle_host_icmp(Context* ctx, IcmpFlow* flow) {
icmp_destroy_flow(ctx, flow);
return;
}
if (retval == 0) {
return;
}
for (int i = 0; i < retval; ++i) {
uint8_t* data_ptr = bufs[i];
size_t recv_len = msgs[i].msg_len;
struct sockaddr_storage* src_addr_storage = &src_addrs[i];
uint8_t* data_ptr = ctx->recvmmsg_bufs[i];
size_t recv_len = ctx->recvmmsg_msgs[i].msg_len;
const struct sockaddr_storage* src_addr_storage = &ctx->recvmmsg_addrs[i];
if (flow->is_ipv6) {
if (flow->header.is_ipv6) {
if ((size_t)recv_len >= sizeof(icmp6_hdr)) {
icmp6_hdr* r_icmp6 = reinterpret_cast<icmp6_hdr*>(data_ptr);
icmp6_hdr icmp6;
memcpy(&icmp6, data_ptr, sizeof(icmp6));
const uint8_t* icmp_payload = data_ptr + sizeof(icmp6_hdr);
size_t icmp_payload_len = recv_len - sizeof(icmp6_hdr);
uint8_t saddr6[IPV6_ADDR_LEN];
if (flow->is_redirected) {
memcpy(saddr6, flow->orig_dest_ip6, sizeof(saddr6));
if (flow->header.is_redirected) {
memcpy(saddr6, flow->header.orig_dest_ip6, sizeof(saddr6));
} else {
struct sockaddr_in6* src6 =
reinterpret_cast<struct sockaddr_in6*>(
const struct sockaddr_in6* src6 =
reinterpret_cast<const struct sockaddr_in6*>(
src_addr_storage);
memcpy(saddr6, &src6->sin6_addr, sizeof(saddr6));
}
icmp_send_packet6(ctx, saddr6, flow->key6.saddr6, r_icmp6->type,
r_icmp6->code, flow->key6.id, r_icmp6->seq, icmp_payload,
icmp_payload_len);
if (!icmp_send_packet6(ctx, saddr6, flow->header.icmp_key6.saddr6,
icmp6.type, icmp6.code, flow->header.icmp_key6.id,
icmp6.seq, icmp_payload, icmp_payload_len)) {
LOG_W("handle_host_icmp: failed to send ICMPv6 packet to "
"guest");
}
}
} else {
struct sockaddr_in* src_addr =
reinterpret_cast<struct sockaddr_in*>(src_addr_storage);
uint32_t saddr =
flow->is_redirected ? flow->orig_dest_ip4 : src_addr->sin_addr.s_addr;
uint32_t daddr = flow->key4.saddr4;
const struct sockaddr_in* src_addr =
reinterpret_cast<const struct sockaddr_in*>(src_addr_storage);
uint32_t saddr = flow->header.is_redirected ? flow->header.orig_dest_ip4
: src_addr->sin_addr.s_addr;
uint32_t daddr = flow->header.icmp_key4.saddr4;
if ((size_t)recv_len >= sizeof(icmp4_hdr)) {
icmp4_hdr* r_icmp = reinterpret_cast<icmp4_hdr*>(data_ptr);
icmp4_hdr icmp4;
memcpy(&icmp4, data_ptr, sizeof(icmp4));
const uint8_t* icmp_payload = data_ptr + sizeof(icmp4_hdr);
size_t icmp_payload_len = recv_len - sizeof(icmp4_hdr);
icmp_send_packet4(ctx, saddr, daddr, r_icmp->type, r_icmp->code,
flow->key4.id, r_icmp->seq, icmp_payload, icmp_payload_len);
if (!icmp_send_packet4(ctx, saddr, daddr, icmp4.type, icmp4.code,
flow->header.icmp_key4.id, icmp4.seq, icmp_payload,
icmp_payload_len)) {
LOG_W("handle_host_icmp: failed to send ICMP packet to "
"guest");
}
}
}
}
}
void IcmpFlow::handle_host_event(Context* ctx, int fd, uint32_t events) {
if (fd == this->host_fd) {
handle_host_icmp(ctx, this);
void icmp_handle_host_event(Context* ctx, IcmpFlow* flow, int fd, uint32_t events) {
if (fd == flow->header.host_fd) {
handle_host_icmp(ctx, flow);
}
}
bool IcmpFlow::is_stale(time_t now) const {
return (now - this->last_active) > 10;
bool icmp_is_stale(const IcmpFlow* flow, time_t now) {
return (now - flow->header.last_active) > 10;
}
void IcmpFlow::destroy(Context* ctx) {
if (is_ipv6) {
LOG_D("GC: stale ICMP flow (IPv6, id=%u)", ntohs(key6.id));
void icmp_destroy(Context* ctx, IcmpFlow* flow) {
if (flow->header.is_ipv6) {
LOG_D("GC: stale ICMP flow (IPv6, id=%u)", ntohs(flow->header.icmp_key6.id));
} else {
LOG_D("GC: stale ICMP flow (id=%u)", ntohs(key4.id));
LOG_D("GC: stale ICMP flow (id=%u)", ntohs(flow->header.icmp_key4.id));
}
icmp_destroy_flow(ctx, this);
icmp_destroy_flow(ctx, flow);
}
} /* namespace nstun */

6
nstun/icmp.h
View File

@@ -4,14 +4,12 @@
#include <stddef.h>
#include <stdint.h>
#include <span>
#include "core.h"
namespace nstun {
void handle_icmp4(Context* ctx, const ip4_hdr* ip, std::span<const uint8_t> payload);
void handle_icmp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payload);
void handle_icmp4(Context* ctx, const ip4_hdr* ip, const uint8_t* payload, size_t len);
void handle_icmp6(Context* ctx, const ip6_hdr* ip, const uint8_t* payload, size_t len);
void send_icmp4_error(
Context* ctx, const ip4_hdr* req_ip, size_t tot_len, uint8_t type, uint8_t code);

64
nstun/iface.cc
View File

@@ -27,7 +27,18 @@ struct in6_ifreq {
};
bool configIface(nsj_t* nsj) {
int sock = socket(AF_INET, SOCK_STREAM, IPPROTO_IP);
struct ifreq ifr = {};
struct in_addr addr;
struct rtentry rt = {};
char rt_dev[IFNAMSIZ];
if (nsj->njc.user_net().ns_iface().length() >= IFNAMSIZ) {
LOG_E("Interface name '%s' is too long (max %d)",
nsj->njc.user_net().ns_iface().c_str(), IFNAMSIZ - 1);
return false;
}
int sock = socket(AF_INET, SOCK_STREAM | SOCK_CLOEXEC, IPPROTO_IP);
if (sock == -1) {
PLOG_E("socket(AF_INET, SOCK_STREAM, IPPROTO_IP)");
return false;
@@ -36,12 +47,7 @@ bool configIface(nsj_t* nsj) {
close(sock);
};
struct ifreq ifr = {};
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", nsj->njc.user_net().ns_iface().c_str());
struct in_addr addr;
struct sockaddr_in* sa = (struct sockaddr_in*)(&ifr.ifr_addr);
/* Set IP Address */
if (!nsj->njc.user_net().ip4().empty()) {
if (inet_pton(AF_INET, nsj->njc.user_net().ip4().c_str(), &addr) != 1) {
@@ -49,8 +55,10 @@ bool configIface(nsj_t* nsj) {
nsj->njc.user_net().ip4().c_str());
return false;
}
sa->sin_family = AF_INET;
sa->sin_addr = addr;
struct sockaddr_in sa = {};
sa.sin_family = AF_INET;
sa.sin_addr = addr;
memcpy(&ifr.ifr_addr, &sa, sizeof(sa));
if (ioctl(sock, SIOCSIFADDR, &ifr) == -1) {
PLOG_E("ioctl(SIOCSIFADDR, '%s')", nsj->njc.user_net().ip4().c_str());
return false;
@@ -59,14 +67,17 @@ bool configIface(nsj_t* nsj) {
/* Set Point-to-Point Destination Address (Host/GW) */
if (!nsj->njc.user_net().gw4().empty()) {
struct sockaddr_in* dst = (struct sockaddr_in*)(&ifr.ifr_dstaddr);
ifr = {};
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", nsj->njc.user_net().ns_iface().c_str());
if (inet_pton(AF_INET, nsj->njc.user_net().gw4().c_str(), &addr) != 1) {
LOG_E("Cannot convert '%s' into an IPv4 GW address",
nsj->njc.user_net().gw4().c_str());
return false;
}
dst->sin_family = AF_INET;
dst->sin_addr = addr;
struct sockaddr_in dst_sa = {};
dst_sa.sin_family = AF_INET;
dst_sa.sin_addr = addr;
memcpy(&ifr.ifr_dstaddr, &dst_sa, sizeof(dst_sa));
if (ioctl(sock, SIOCSIFDSTADDR, &ifr) == -1) {
PLOG_E("ioctl(SIOCSIFDSTADDR, '%s')", nsj->njc.user_net().gw4().c_str());
return false;
@@ -74,9 +85,12 @@ bool configIface(nsj_t* nsj) {
}
/* Set Netmask to /32 for PtP link */
struct sockaddr_in* netmask = (struct sockaddr_in*)(&ifr.ifr_netmask);
netmask->sin_family = AF_INET;
netmask->sin_addr.s_addr = 0xFFFFFFFF; // 255.255.255.255
ifr = {};
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", nsj->njc.user_net().ns_iface().c_str());
struct sockaddr_in netmask_sa = {};
netmask_sa.sin_family = AF_INET;
netmask_sa.sin_addr.s_addr = 0xFFFFFFFF; /* 255.255.255.255 */
memcpy(&ifr.ifr_netmask, &netmask_sa, sizeof(netmask_sa));
if (ioctl(sock, SIOCSIFNETMASK, &ifr) == -1) {
PLOG_E("ioctl(SIOCSIFNETMASK, 255.255.255.255)");
return false;
@@ -93,24 +107,26 @@ bool configIface(nsj_t* nsj) {
return false;
}
ifr = {};
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", nsj->njc.user_net().ns_iface().c_str());
ifr.ifr_mtu = NSTUN_MTU;
if (ioctl(sock, SIOCSIFMTU, &ifr) == -1) {
PLOG_W("ioctl(SIOCSIFMTU, %zu)", NSTUN_MTU);
PLOG_E("ioctl(SIOCSIFMTU, %zu)", NSTUN_MTU);
return false;
}
/* Add default route out of interface */
struct rtentry rt = {};
struct sockaddr_in* sdest = (struct sockaddr_in*)(&rt.rt_dst);
struct sockaddr_in* smask = (struct sockaddr_in*)(&rt.rt_genmask);
struct sockaddr_in sdest_sa = {};
sdest_sa.sin_family = AF_INET;
sdest_sa.sin_addr.s_addr = INADDR_ANY;
memcpy(&rt.rt_dst, &sdest_sa, sizeof(sdest_sa));
sdest->sin_family = AF_INET;
sdest->sin_addr.s_addr = INADDR_ANY;
smask->sin_family = AF_INET;
smask->sin_addr.s_addr = INADDR_ANY;
struct sockaddr_in smask_sa = {};
smask_sa.sin_family = AF_INET;
smask_sa.sin_addr.s_addr = INADDR_ANY;
memcpy(&rt.rt_genmask, &smask_sa, sizeof(smask_sa));
rt.rt_flags = RTF_UP; /* Device route, no gateway necessary for PtP */
char rt_dev[IFNAMSIZ];
snprintf(rt_dev, sizeof(rt_dev), "%s", nsj->njc.user_net().ns_iface().c_str());
rt.rt_dev = rt_dev;
@@ -123,7 +139,7 @@ bool configIface(nsj_t* nsj) {
/* Configure IPv6 address and route */
if (!nsj->njc.user_net().ip6().empty()) {
int sock6 = socket(AF_INET6, SOCK_DGRAM, 0);
int sock6 = socket(AF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
if (sock6 == -1) {
PLOG_E("socket(AF_INET6, SOCK_DGRAM)");
return false;

215
nstun/ip.cc
View File

@@ -1,6 +1,8 @@
#include "ip.h"
#include <arpa/inet.h>
#include <netinet/in.h>
#include <stdio.h>
#include <string.h>
#include "core.h"
@@ -62,42 +64,61 @@ static int skip_ipv6_ext_headers(int next_header, const uint8_t*& ptr, size_t& r
return -1; /* Extension header chain too deep */
}
void handle_ip4(Context* ctx, std::span<const uint8_t> payload) {
if (payload.size() < sizeof(ip4_hdr)) {
static bool extract_l4_ports(
int proto, const uint8_t* payload, size_t len, uint16_t* src_port, uint16_t* dest_port) {
if (proto == IPPROTO_TCP && len >= sizeof(tcp_hdr)) {
tcp_hdr tcp;
memcpy(&tcp, payload, sizeof(tcp));
*src_port = ntohs(tcp.source);
*dest_port = ntohs(tcp.dest);
return true;
} else if (proto == IPPROTO_UDP && len >= sizeof(udp_hdr)) {
udp_hdr udp;
memcpy(&udp, payload, sizeof(udp));
*src_port = ntohs(udp.source);
*dest_port = ntohs(udp.dest);
return true;
}
return false;
}
void handle_ip4(Context* ctx, const uint8_t* payload, size_t len) {
if (len < sizeof(ip4_hdr)) {
return;
}
const ip4_hdr* ip = reinterpret_cast<const ip4_hdr*>(payload.data());
uint8_t ihl = ip4_ihl(ip) * 4;
ip4_hdr ip;
memcpy(&ip, payload, sizeof(ip));
uint8_t ihl = ip4_ihl(&ip) * 4;
if (ihl < sizeof(ip4_hdr) || ihl > payload.size()) {
if (ihl < sizeof(ip4_hdr) || ihl > len) {
LOG_D("Invalid IPv4 IHL");
return;
}
uint16_t tot_len = ntohs(ip->tot_len);
if (tot_len < ihl || tot_len > payload.size()) {
uint16_t tot_len = ntohs(ip.tot_len);
if (tot_len < ihl || tot_len > len) {
LOG_D("Invalid IPv4 tot_len");
return;
}
const uint8_t* l4_payload = payload.data() + ihl;
const uint8_t* l4_payload = payload + ihl;
size_t l4_len = tot_len - ihl;
/* Drop IP fragments: nstun does not reassemble, and non-first
* fragments have no L4 header - parsing them would bypass rules */
if (ntohs(ip->frag_off) & 0x3FFF) {
if (ntohs(ip.frag_off) & 0x3FFF) {
LOG_W("Dropping IPv4 fragment");
return;
}
/* Validate IPv4 header checksum */
if (compute_checksum(ip, ihl) != 0) {
if (compute_checksum(payload, ihl) != 0) {
LOG_W("Invalid IPv4 header checksum, dropping");
return;
}
if (ip->saddr != ctx->guest_ip4 && ip->saddr != 0) {
if (ip.saddr != ctx->guest_ip4 && ip.saddr != 0) {
LOG_W("Dropping packet with invalid source IP");
return;
}
@@ -106,72 +127,86 @@ void handle_ip4(Context* ctx, std::span<const uint8_t> payload) {
* This is the single authoritative check - L4 handlers rely on this
* and do NOT duplicate it. Redirect rules in policy may still target
* loopback intentionally (admin-controlled). */
if (IN_LOOPBACK(ntohl(ip->daddr)) || ip->daddr == htonl(INADDR_ANY) ||
ip->daddr == htonl(INADDR_BROADCAST)) {
LOG_W("Dropping packet destined to loopback, ANY, or broadcast: %s",
ip4_to_string(ip->daddr).c_str());
if (IN_LOOPBACK(ntohl(ip.daddr)) || ip.daddr == htonl(INADDR_ANY) ||
ip.daddr == htonl(INADDR_BROADCAST)) {
char daddr_str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ip.daddr, daddr_str, sizeof(daddr_str));
LOG_W("Dropping packet destined to loopback, ANY, or broadcast: %s", daddr_str);
return;
}
uint16_t src_port = 0, dest_port = 0;
auto l4_span = payload.subspan(ihl, l4_len);
if (ip->protocol == IPPROTO_TCP && l4_span.size() >= sizeof(tcp_hdr)) {
const tcp_hdr* tcp = reinterpret_cast<const tcp_hdr*>(l4_span.data());
src_port = ntohs(tcp->source);
dest_port = ntohs(tcp->dest);
} else if (ip->protocol == IPPROTO_UDP && l4_span.size() >= sizeof(udp_hdr)) {
const udp_hdr* udp = reinterpret_cast<const udp_hdr*>(l4_span.data());
src_port = ntohs(udp->source);
dest_port = ntohs(udp->dest);
extract_l4_ports(ip.protocol, l4_payload, l4_len, &src_port, &dest_port);
if (logs::getLogLevel() <= logs::DEBUG) {
char saddr_str[INET_ADDRSTRLEN];
char daddr_str[INET_ADDRSTRLEN];
inet_ntop(AF_INET, &ip.saddr, saddr_str, sizeof(saddr_str));
inet_ntop(AF_INET, &ip.daddr, daddr_str, sizeof(daddr_str));
if (src_port != 0 && dest_port != 0) {
LOG_D("IP packet: proto=%u, %s:%u -> %s:%u, len=%zu", ip.protocol,
saddr_str, src_port, daddr_str, dest_port, l4_len);
} else {
LOG_D("IP packet: proto=%u, %s -> %s, len=%zu", ip.protocol, saddr_str,
daddr_str, l4_len);
}
}
if (src_port != 0 && dest_port != 0) {
LOG_D("IP packet: proto=%u, %s:%u -> %s:%u, len=%zu", ip->protocol,
ip4_to_string(ip->saddr).c_str(), src_port, ip4_to_string(ip->daddr).c_str(),
dest_port, l4_len);
} else {
LOG_D("IP packet: proto=%u, %s -> %s, len=%zu", ip->protocol,
ip4_to_string(ip->saddr).c_str(), ip4_to_string(ip->daddr).c_str(), l4_len);
}
switch (ip->protocol) {
switch (ip.protocol) {
case IPPROTO_ICMP:
handle_icmp4(ctx, ip, payload.subspan(ihl, l4_len));
LOG_D("Calling handle_icmp4");
LOG_D("about to call handle_icmp4: ctx=%p, ip=%p, l4=%p, len=%zu", (void*)ctx,
(void*)&ip, (void*)l4_payload, l4_len);
handle_icmp4(ctx, &ip, l4_payload, l4_len);
break;
case IPPROTO_UDP:
handle_udp4(ctx, ip, payload.subspan(ihl, l4_len));
handle_udp4(ctx, &ip, l4_payload, l4_len);
break;
case IPPROTO_TCP:
handle_tcp4(ctx, ip, payload.subspan(ihl, l4_len));
handle_tcp4(ctx, &ip, l4_payload, l4_len);
break;
default:
LOG_D("Unknown IPv4 protocol: %u", ip->protocol);
LOG_D("Unknown IPv4 protocol: %u", ip.protocol);
break;
}
}
void handle_ip6(Context* ctx, std::span<const uint8_t> payload) {
if (payload.size() < sizeof(ip6_hdr)) {
void handle_ip6(Context* ctx, const uint8_t* payload, size_t len) {
if (len < sizeof(ip6_hdr)) {
return;
}
const ip6_hdr* ip6 = reinterpret_cast<const ip6_hdr*>(payload.data());
uint16_t payload_len = ntohs(ip6->payload_len);
ip6_hdr ip6;
memcpy(&ip6, payload, sizeof(ip6));
uint16_t payload_len = ntohs(ip6.payload_len);
if (payload_len + sizeof(ip6_hdr) > payload.size()) {
if (payload_len + sizeof(ip6_hdr) > len) {
LOG_D("Invalid IPv6 payload_len");
return;
}
/* Source IP filtering */
if (memcmp(ip6->saddr, ctx->guest_ip6, IPV6_ADDR_LEN) != 0) {
if (IN6_IS_ADDR_LINKLOCAL((const struct in6_addr*)ip6->saddr) ||
IN6_IS_ADDR_SITELOCAL((const struct in6_addr*)ip6->saddr)) {
LOG_D("Dropping IPv6 packet with link/site-local source address: %s",
ip6_to_string(ip6->saddr).c_str());
if (memcmp(ip6.saddr, ctx->guest_ip6, IPV6_ADDR_LEN) != 0) {
struct in6_addr saddr;
memcpy(&saddr, ip6.saddr, sizeof(saddr));
if (IN6_IS_ADDR_LINKLOCAL(&saddr)) {
/* Allow link-local addresses from guest on the local link */
if (logs::getLogLevel() <= logs::DEBUG) {
char saddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.saddr, saddr_str, sizeof(saddr_str));
LOG_D("Allowing link-local source address: %s", saddr_str);
}
} else if (IN6_IS_ADDR_SITELOCAL(&saddr)) {
if (logs::getLogLevel() <= logs::DEBUG) {
char saddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.saddr, saddr_str, sizeof(saddr_str));
LOG_D("Dropping IPv6 packet with site-local source address: %s",
saddr_str);
}
return;
} else {
LOG_W("Dropping IPv6 packet with spoofed source address: %s",
ip6_to_string(ip6->saddr).c_str());
char saddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.saddr, saddr_str, sizeof(saddr_str));
LOG_W("Dropping IPv6 packet with spoofed source address: %s", saddr_str);
return;
}
}
@@ -180,24 +215,40 @@ void handle_ip6(Context* ctx, std::span<const uint8_t> payload) {
* This is the single authoritative check - L4 handlers rely on this
* and do NOT duplicate it. Redirect rules in policy may still target
* ::1 intentionally (admin-controlled). */
if (IN6_IS_ADDR_LOOPBACK((const struct in6_addr*)ip6->daddr)) {
LOG_D("Dropping IPv6 packet to loopback: %s", ip6_to_string(ip6->daddr).c_str());
struct in6_addr daddr;
memcpy(&daddr, ip6.daddr, sizeof(daddr));
if (IN6_IS_ADDR_LOOPBACK(&daddr)) {
if (logs::getLogLevel() <= logs::DEBUG) {
char daddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.daddr, daddr_str, sizeof(daddr_str));
LOG_D("Dropping IPv6 packet to loopback: %s", daddr_str);
}
return;
}
if (IN6_IS_ADDR_V4MAPPED((const struct in6_addr*)ip6->daddr)) {
LOG_D("Dropping IPv6 packet to v4-mapped address (use IPv4 directly): %s",
ip6_to_string(ip6->daddr).c_str());
if (IN6_IS_ADDR_V4MAPPED(&daddr)) {
if (logs::getLogLevel() <= logs::DEBUG) {
char daddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.daddr, daddr_str, sizeof(daddr_str));
LOG_D("Dropping IPv6 packet to v4-mapped address (use IPv4 directly): %s",
daddr_str);
}
return;
}
if (IN6_IS_ADDR_V4COMPAT((const struct in6_addr*)ip6->daddr)) {
LOG_D("Dropping IPv6 packet to v4-compatible address (deprecated): %s",
ip6_to_string(ip6->daddr).c_str());
if (IN6_IS_ADDR_V4COMPAT(&daddr)) {
if (logs::getLogLevel() <= logs::DEBUG) {
char daddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.daddr, daddr_str, sizeof(daddr_str));
LOG_D("Dropping IPv6 packet to v4-compatible address (deprecated): %s",
daddr_str);
}
return;
}
if (IN6_IS_ADDR_LINKLOCAL((const struct in6_addr*)ip6->daddr) ||
IN6_IS_ADDR_SITELOCAL((const struct in6_addr*)ip6->daddr)) {
LOG_D("Dropping IPv6 packet to link/site-local address: %s",
ip6_to_string(ip6->daddr).c_str());
if (IN6_IS_ADDR_LINKLOCAL(&daddr) || IN6_IS_ADDR_SITELOCAL(&daddr)) {
if (logs::getLogLevel() <= logs::DEBUG) {
char daddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.daddr, daddr_str, sizeof(daddr_str));
LOG_D("Dropping IPv6 packet to link/site-local address: %s", daddr_str);
}
return;
}
/*
@@ -207,45 +258,41 @@ void handle_ip6(Context* ctx, std::span<const uint8_t> payload) {
* Each header's "Next Header" field identifies what follows.
* We only need to find the L4 header, not process the extensions.
*/
const uint8_t* l4_payload = payload.data() + sizeof(ip6_hdr);
const uint8_t* l4_payload = payload + sizeof(ip6_hdr);
size_t remaining = payload_len;
int l4_proto = skip_ipv6_ext_headers(ip6->next_header, l4_payload, remaining);
int l4_proto = skip_ipv6_ext_headers(ip6.next_header, l4_payload, remaining);
if (l4_proto < 0) {
LOG_D("Failed to parse IPv6 extension headers");
return;
}
uint16_t src_port = 0, dest_port = 0;
if (l4_proto == IPPROTO_TCP && remaining >= sizeof(tcp_hdr)) {
const tcp_hdr* tcp = reinterpret_cast<const tcp_hdr*>(l4_payload);
src_port = ntohs(tcp->source);
dest_port = ntohs(tcp->dest);
} else if (l4_proto == IPPROTO_UDP && remaining >= sizeof(udp_hdr)) {
const udp_hdr* udp = reinterpret_cast<const udp_hdr*>(l4_payload);
src_port = ntohs(udp->source);
dest_port = ntohs(udp->dest);
}
extract_l4_ports(l4_proto, l4_payload, remaining, &src_port, &dest_port);
if (src_port != 0 && dest_port != 0) {
LOG_D("IPv6 packet: next_header=%u, %s:%u -> %s:%u, len=%u", l4_proto,
ip6_to_string(ip6->saddr).c_str(), src_port, ip6_to_string(ip6->daddr).c_str(),
dest_port, payload_len);
} else {
LOG_D("IPv6 packet: next_header=%u, %s -> %s, len=%u", l4_proto,
ip6_to_string(ip6->saddr).c_str(), ip6_to_string(ip6->daddr).c_str(),
payload_len);
if (logs::getLogLevel() <= logs::DEBUG) {
char saddr_str[INET6_ADDRSTRLEN];
char daddr_str[INET6_ADDRSTRLEN];
inet_ntop(AF_INET6, ip6.saddr, saddr_str, sizeof(saddr_str));
inet_ntop(AF_INET6, ip6.daddr, daddr_str, sizeof(daddr_str));
if (src_port != 0 && dest_port != 0) {
LOG_D("IPv6 packet: next_header=%u, %s:%u -> %s:%u, len=%u", l4_proto,
saddr_str, src_port, daddr_str, dest_port, payload_len);
} else {
LOG_D("IPv6 packet: next_header=%u, %s -> %s, len=%u", l4_proto, saddr_str,
daddr_str, payload_len);
}
}
switch (l4_proto) {
case IPPROTO_ICMPV6:
handle_icmp6(ctx, ip6, std::span<const uint8_t>(l4_payload, remaining));
handle_icmp6(ctx, &ip6, l4_payload, remaining);
break;
case IPPROTO_UDP:
handle_udp6(ctx, ip6, std::span<const uint8_t>(l4_payload, remaining));
handle_udp6(ctx, &ip6, l4_payload, remaining);
break;
case IPPROTO_TCP:
handle_tcp6(ctx, ip6, std::span<const uint8_t>(l4_payload, remaining));
handle_tcp6(ctx, &ip6, l4_payload, remaining);
break;
default:
LOG_D("Unknown IPv6 next_header: %u", l4_proto);

6
nstun/ip.h
View File

@@ -4,14 +4,12 @@
#include <stddef.h>
#include <stdint.h>
#include <span>
#include "core.h"
namespace nstun {
void handle_ip4(Context* ctx, std::span<const uint8_t> payload);
void handle_ip6(Context* ctx, std::span<const uint8_t> payload);
void handle_ip4(Context* ctx, const uint8_t* payload, size_t len);
void handle_ip6(Context* ctx, const uint8_t* payload, size_t len);
} // namespace nstun

5
nstun/net_defs.h
View File

@@ -1,14 +1,14 @@
#ifndef NSTUN_NET_DEFS_H_
#define NSTUN_NET_DEFS_H_
#include <string>
#include <arpa/inet.h>
#include <netinet/in.h>
#include <stddef.h>
#include <stdint.h>
#include <string.h>
#include <string>
/* From <linux/in.h>, can't include directly due to conflicts with <netinet/in.h> */
#ifndef IN_LOOPBACK
#define IN_LOOPBACK(a) ((((long int)(a)) & 0xff000000) == 0x7f000000)
@@ -110,7 +110,6 @@ inline uint8_t ip_version(const uint8_t* ptr) {
return ptr[0] >> 4;
}
inline uint8_t ip4_version(const ip4_hdr* h) {
return h->ihl_version >> 4;
}

866
nstun/nstun.cc
View File

@@ -9,6 +9,7 @@
#include <netlink/addr.h>
#include <netlink/netlink.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -18,6 +19,7 @@
#include <sys/wait.h>
#include <unistd.h>
#include <new>
#include <thread>
#include "core.h"
@@ -26,6 +28,8 @@
#include "ip.h"
#include "logs.h"
#include "macros.h"
#include "monitor.h"
#include "nsjail.h"
#include "policy.h"
#include "tcp.h"
#include "tun.h"
@@ -34,52 +38,136 @@
namespace nstun {
Context::~Context() {
/* Owning maps (ipv4_udp_flows_by_key, etc.) will self-clean via std::unique_ptr */
for (auto& [fd, _] : host_listener_fd_to_rule) {
::close(fd);
bool icmp_is_stale(const IcmpFlow* flow, time_t now);
void icmp_destroy(Context* ctx, IcmpFlow* flow);
void icmp_handle_host_event(Context* ctx, IcmpFlow* flow, int fd, uint32_t events);
static void context_cleanup(Context* ctx) {
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
if (ctx->c_ipv4_udp_flows[i].header.active) {
udp_destroy_flow(ctx, &ctx->c_ipv4_udp_flows[i]);
}
if (ctx->c_ipv6_udp_flows[i].header.active) {
udp_destroy_flow(ctx, &ctx->c_ipv6_udp_flows[i]);
}
if (ctx->c_ipv4_tcp_flows[i].header.active) {
tcp_destroy_flow(ctx, &ctx->c_ipv4_tcp_flows[i]);
}
if (ctx->c_ipv6_tcp_flows[i].header.active) {
tcp_destroy_flow(ctx, &ctx->c_ipv6_tcp_flows[i]);
}
if (ctx->c_ipv4_icmp_flows[i].header.active) {
icmp_destroy(ctx, &ctx->c_ipv4_icmp_flows[i]);
}
if (ctx->c_ipv6_icmp_flows[i].header.active) {
icmp_destroy(ctx, &ctx->c_ipv6_icmp_flows[i]);
}
}
for (size_t i = 0; i < ctx->num_c_host_listener_rules; ++i) {
monitor::removeFd(ctx->c_host_listener_rules[i].fd);
close(ctx->c_host_listener_rules[i].fd);
}
ctx->num_c_host_listener_rules = 0;
if (ctx->tap_fd != -1) {
monitor::removeFd(ctx->tap_fd);
close(ctx->tap_fd);
}
}
static void gc_destroy_tcp_flow(Context* ctx, TcpFlow* flow) {
if (flow->header.is_ipv6) {
LOG_D("GC: stale TCP flow (IPv6, sport=%u, state=%d)",
ntohs(flow->header.key6.sport), static_cast<int>(flow->tcp_state));
} else {
LOG_D("GC: stale TCP flow (sport=%u, state=%d)", ntohs(flow->header.key4.sport),
static_cast<int>(flow->tcp_state));
}
tcp_destroy_flow(ctx, flow);
}
static void gc_tcp_flows(Context* ctx, TcpFlow* flows, time_t now) {
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
TcpFlow* flow = &flows[i];
if (flow->header.active) {
tcp_periodic_check(ctx, flow, now);
if (is_stale_tcp(flow, now)) {
gc_destroy_tcp_flow(ctx, flow);
}
}
}
}
static void gc_udp_flows(Context* ctx, UdpFlow* flows, time_t now) {
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
UdpFlow* flow = &flows[i];
if (flow->header.active) {
if (is_stale_udp(flow, now)) {
udp_destroy_flow(ctx, flow);
}
}
}
}
static void gc_icmp_flows(Context* ctx, IcmpFlow* flows, time_t now) {
for (size_t i = 0; i < NSTUN_MAX_FLOWS; ++i) {
IcmpFlow* flow = &flows[i];
if (flow->header.active) {
if (icmp_is_stale(flow, now)) {
icmp_destroy(ctx, flow);
}
}
}
}
static void garbage_collect(Context* ctx) {
time_t now = time(NULL);
time_t now = time(nullptr);
auto do_gc = [&](auto& map) {
std::vector<Flow*> stale_flows;
for (auto const& pair : map) {
Flow* flow = pair.second.get();
flow->periodic_check(ctx, now);
if (flow->is_stale(now)) {
stale_flows.push_back(flow);
}
}
for (Flow* flow : stale_flows) {
flow->destroy(ctx);
}
};
do_gc(ctx->ipv4_tcp_flows_by_key);
do_gc(ctx->ipv4_udp_flows_by_key);
do_gc(ctx->ipv4_icmp_flows_by_key);
do_gc(ctx->ipv6_tcp_flows_by_key);
do_gc(ctx->ipv6_udp_flows_by_key);
do_gc(ctx->ipv6_icmp_flows_by_key);
gc_tcp_flows(ctx, ctx->c_ipv4_tcp_flows, now);
gc_tcp_flows(ctx, ctx->c_ipv6_tcp_flows, now);
gc_udp_flows(ctx, ctx->c_ipv4_udp_flows, now);
gc_udp_flows(ctx, ctx->c_ipv6_udp_flows, now);
gc_icmp_flows(ctx, ctx->c_ipv4_icmp_flows, now);
gc_icmp_flows(ctx, ctx->c_ipv6_icmp_flows, now);
}
static void handle_host_events(Context* ctx, int fd, uint32_t events) {
auto it_listener = ctx->host_listener_fd_to_rule.find(fd);
if (it_listener != ctx->host_listener_fd_to_rule.end()) {
if (it_listener->second.proto == NSTUN_PROTO_TCP) {
handle_host_tcp_accept(ctx, fd, it_listener->second);
} else if (it_listener->second.proto == NSTUN_PROTO_UDP) {
handle_host_udp_accept(ctx, fd, it_listener->second);
void handle_host_events(Context* ctx, int fd, uint32_t events) {
LOG_D("handle_host_events: fd=%d, events=0x%x", fd, events);
for (size_t i = 0; i < ctx->num_c_host_listener_rules; ++i) {
if (ctx->c_host_listener_rules[i].fd == fd) {
const nstun_rule_t& rule = ctx->c_host_listener_rules[i].rule;
switch (rule.proto) {
case NSTUN_PROTO_TCP:
handle_host_tcp_accept(ctx, fd, rule);
break;
case NSTUN_PROTO_UDP:
handle_host_udp_accept(ctx, fd, rule);
break;
default:
break;
}
return;
}
}
TcpFlow* tcp_flow = get_tcp_flow_by_fd(ctx, fd);
if (tcp_flow) {
handle_host_tcp_event(ctx, tcp_flow, fd, events);
return;
}
UdpFlow* udp_flow = get_udp_flow_by_fd(ctx, fd);
if (udp_flow) {
handle_host_udp_event(ctx, udp_flow, fd, events);
return;
}
IcmpFlow* icmp_flow = get_icmp_flow_by_fd(ctx, fd);
if (icmp_flow) {
icmp_handle_host_event(ctx, icmp_flow, fd, events);
return;
}
auto it = ctx->flows_by_fd.find(fd);
if (it != ctx->flows_by_fd.end()) {
it->second->handle_host_event(ctx, fd, events);
if (fd < 0 || fd >= static_cast<int>(NSTUN_MAX_FDS)) {
LOG_W("FD %d is out of bounds for lookup table (max %zu)", fd, NSTUN_MAX_FDS);
return;
}
@@ -88,65 +176,15 @@ static void handle_host_events(Context* ctx, int fd, uint32_t events) {
LOG_D("Stale epoll event for fd %d (already closed), skipping", fd);
}
static void networkLoop(Context* ctx) {
LOG_D("nstun network loop started on tap_fd=%d", ctx->tap_fd);
defer {
close(ctx->tap_fd);
close(ctx->epoll_fd);
delete ctx;
};
struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = ctx->tap_fd}};
if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, ctx->tap_fd, &ev) == -1) {
PLOG_E("epoll_ctl(EPOLL_CTL_ADD, tap_fd)");
return;
}
/* TUN frames: 4-byte header + up to NSTUN_MTU bytes of L3 payload */
static constexpr size_t TUN_FRAME_BUF_SIZE = NSTUN_MTU + 4;
auto buf = std::make_unique<uint8_t[]>(TUN_FRAME_BUF_SIZE);
struct epoll_event events[64];
time_t last_gc = time(NULL);
while (true) {
int nfds = epoll_wait(ctx->epoll_fd, events, 64, 1000); /* 1s timeout */
if (nfds == -1) {
if (errno == EINTR) continue;
PLOG_E("epoll_wait");
break;
}
time_t now = time(NULL);
if (now - last_gc >= 2) {
garbage_collect(ctx);
last_gc = now;
}
for (int i = 0; i < nfds; ++i) {
int fd = events[i].data.fd;
if (fd == ctx->tap_fd) {
ssize_t n = TEMP_FAILURE_RETRY(
read(ctx->tap_fd, buf.get(), TUN_FRAME_BUF_SIZE));
if (n <= 0) {
if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
continue;
}
PLOG_E("read(tap_fd) failed or EOF");
return;
}
handle_tun_frame(ctx, buf.get(), n);
} else {
handle_host_events(ctx, fd, events[i].events);
}
}
}
void host_callback(int fd, uint32_t events, void* data) {
Context* ctx = static_cast<Context*>(data);
LOG_D("host_callback: ctx=%p", ctx);
handle_host_events(ctx, fd, events);
}
} /* namespace nstun */
bool nstun_init_child(int sock, nsj_t* nsj) {
bool nstun_init_child(int ipc_fd, nsj_t* nsj) {
/* Create TUN device. */
int tap_fd = open("/dev/net/tun", O_RDWR | O_CLOEXEC | O_NONBLOCK);
if (tap_fd < 0) {
@@ -154,252 +192,315 @@ bool nstun_init_child(int sock, nsj_t* nsj) {
return false;
}
bool success = false;
defer {
if (!success) {
close(tap_fd);
}
};
struct ifreq ifr = {};
ifr.ifr_flags = IFF_TUN | IFF_NO_PI; /* TUN, no packet info */
snprintf(ifr.ifr_name, IFNAMSIZ, "%s", nsj->njc.user_net().ns_iface().c_str());
if (ioctl(tap_fd, TUNSETIFF, &ifr) < 0) {
PLOG_E("ioctl(TUNSETIFF)");
close(tap_fd);
return false;
}
/* Configure IP, MAC, UP, route. */
if (!nstun::configIface(nsj)) {
LOG_E("nstun::configIface() failed");
close(tap_fd);
return false;
}
/* Send FD to parent */
if (!util::sendFd(sock, tap_fd)) {
PLOG_E("util::sendFd(tap_fd)");
if (!util::sendMsg(ipc_fd, monitor::MSG_TAG_TAP, tap_fd)) {
PLOG_E("util::sendMsg(tap_fd)");
close(tap_fd);
return false;
}
success = true;
close(tap_fd);
return true;
}
bool nstun_init_parent(int sock, nsj_t* nsj) {
int tap_fd = util::recvFd(sock);
if (tap_fd < 0) {
LOG_E("Failed to receive TAP fd from child");
static void tapCb(int fd, uint32_t /* events */, void* data) {
nstun::Context* ctx = static_cast<nstun::Context*>(data);
/* Rule 21: Avoid infinite loop to prevent event starvation.
* Level-triggered epoll will wake us up again if more data is available. */
ssize_t n = TEMP_FAILURE_RETRY(read(fd, ctx->tun_buf, nstun::NSTUN_MTU + 4));
if (n <= 0) {
if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK)) {
return;
}
PLOG_E("read(tap_fd) failed or EOF, removing FD");
monitor::removeFd(fd);
close(fd);
ctx->tap_fd = -1;
return;
}
handle_tun_frame(ctx, ctx->tun_buf, n);
}
static thread_local nstun::Context* tls_nstun_ctx = nullptr;
void nstun_periodic() {
if (tls_nstun_ctx) {
garbage_collect(tls_nstun_ctx);
}
}
static bool assign_ip(const std::string& str, uint32_t* ip) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET, &addr) != 0) {
LOG_E("Failed to parse IP string: %s", str.c_str());
return false;
}
LOG_I("nstun initialized successfully, tap_fd=%d", tap_fd);
std::unique_ptr<nstun::Context> ctx = std::make_unique<nstun::Context>();
ctx->epoll_fd = -1; /* Not used yet */
ctx->tap_fd = tap_fd;
ctx->nsj = nsj;
auto assign_ip = [](const std::string& str, uint32_t* ip) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET, &addr) == 0) {
if (nl_addr_get_len(addr) == 4) {
memcpy(ip, nl_addr_get_binary_addr(addr), 4);
}
nl_addr_put(addr);
}
};
if (!nsj->njc.user_net().ip4().empty()) {
assign_ip(nsj->njc.user_net().ip4(), &ctx->guest_ip4);
}
if (!nsj->njc.user_net().gw4().empty()) {
assign_ip(nsj->njc.user_net().gw4(), &ctx->host_ip4);
}
if (!nsj->njc.user_net().ip6().empty()) {
if (inet_pton(AF_INET6, nsj->njc.user_net().ip6().c_str(), ctx->guest_ip6) != 1) {
LOG_E("Cannot convert '%s' into an IPv6 address",
nsj->njc.user_net().ip6().c_str());
close(tap_fd);
return false;
}
}
if (!nsj->njc.user_net().gw6().empty()) {
if (inet_pton(AF_INET6, nsj->njc.user_net().gw6().c_str(), ctx->host_ip6) != 1) {
LOG_E("Cannot convert '%s' into an IPv6 address",
nsj->njc.user_net().gw6().c_str());
close(tap_fd);
return false;
}
}
auto parse_ip = [](const std::string& str, uint32_t* ip, uint32_t* mask) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET, &addr) == 0) {
if (nl_addr_get_len(addr) == 4) {
memcpy(ip, nl_addr_get_binary_addr(addr), 4);
}
int bits = nl_addr_get_prefixlen(addr);
*mask = (bits == 0) ? 0 : htonl(~((1ULL << (32 - bits)) - 1));
nl_addr_put(addr);
} else {
LOG_E("Failed to parse IP/CIDR string: %s", str.c_str());
}
};
auto parse_ip6 = [](const std::string& str, uint8_t* ip6, uint8_t* mask6) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET6, &addr) == 0) {
if (nl_addr_get_len(addr) == nstun::IPV6_ADDR_LEN) {
memcpy(ip6, nl_addr_get_binary_addr(addr), nstun::IPV6_ADDR_LEN);
}
int bits = nl_addr_get_prefixlen(addr);
memset(mask6, 0, nstun::IPV6_ADDR_LEN);
for (int i = 0; i < (int)nstun::IPV6_ADDR_LEN; i++) {
if (bits >= 8) {
mask6[i] = 0xFF;
bits -= 8;
} else if (bits > 0) {
mask6[i] = (uint8_t)(0xFF << (8 - bits));
bits = 0;
} else {
mask6[i] = 0;
}
}
nl_addr_put(addr);
} else {
LOG_E("Failed to parse IPv6/CIDR string: %s", str.c_str());
}
};
ctx->epoll_fd = epoll_create1(EPOLL_CLOEXEC);
if (ctx->epoll_fd == -1) {
PLOG_E("epoll_create1(EPOLL_CLOEXEC)");
close(ctx->tap_fd);
if (nl_addr_get_len(addr) != 4) {
LOG_E("IP string is not IPv4: %s", str.c_str());
nl_addr_put(addr);
return false;
}
memcpy(ip, nl_addr_get_binary_addr(addr), 4);
nl_addr_put(addr);
return true;
}
auto cleanup_and_fail = [&ctx]() -> bool {
for (auto& [fd, _] : ctx->host_listener_fd_to_rule) {
static bool parse_ip(const std::string& str, uint32_t* ip, uint32_t* mask) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET, &addr) != 0) {
LOG_E("Failed to parse IP/CIDR string: %s", str.c_str());
return false;
}
if (nl_addr_get_len(addr) != 4) {
LOG_E("IP/CIDR string is not IPv4: %s", str.c_str());
nl_addr_put(addr);
return false;
}
memcpy(ip, nl_addr_get_binary_addr(addr), 4);
int bits = nl_addr_get_prefixlen(addr);
*mask = (bits == 0) ? 0 : htonl(~((1ULL << (32 - bits)) - 1));
nl_addr_put(addr);
return true;
}
static bool parse_ip6(const std::string& str, uint8_t* ip6, uint8_t* mask6) {
struct nl_addr* addr;
if (nl_addr_parse(str.c_str(), AF_INET6, &addr) != 0) {
LOG_E("Failed to parse IPv6/CIDR string: %s", str.c_str());
return false;
}
if (nl_addr_get_len(addr) != nstun::IPV6_ADDR_LEN) {
LOG_E("IPv6/CIDR string is not IPv6: %s", str.c_str());
nl_addr_put(addr);
return false;
}
memcpy(ip6, nl_addr_get_binary_addr(addr), nstun::IPV6_ADDR_LEN);
int bits = nl_addr_get_prefixlen(addr);
memset(mask6, 0, nstun::IPV6_ADDR_LEN);
for (size_t i = 0; i < nstun::IPV6_ADDR_LEN; i++) {
if (bits >= 8) {
mask6[i] = 0xFF;
bits -= 8;
} else if (bits > 0) {
mask6[i] = (uint8_t)(0xFF << (8 - bits));
bits = 0;
} else {
mask6[i] = 0;
}
}
nl_addr_put(addr);
return true;
}
static int create_host_listener(int domain, int type, const struct sockaddr* addr,
socklen_t addrlen, nstun::Context* ctx, bool is_tcp, uint32_t port) {
int fd = socket(domain, type | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd == -1) {
PLOG_E("socket() for HOST_TO_GUEST");
return -1;
}
int opt = 1;
if (is_tcp) {
if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &opt, sizeof(opt)) == -1) {
PLOG_W("setsockopt(TCP_NODELAY)");
}
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) == -1) {
PLOG_E("setsockopt(SO_REUSEADDR)");
close(fd);
return -1;
}
if (domain == AF_INET6) {
if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) == -1) {
PLOG_E("setsockopt(IPV6_V6ONLY)");
close(fd);
return -1;
}
close(ctx->epoll_fd);
close(ctx->tap_fd);
return false;
};
}
if (bind(fd, addr, addrlen) == -1) {
PLOG_E("bind() for HOST_TO_GUEST on port %u", port);
close(fd);
return -1;
}
if (is_tcp) {
if (listen(fd, SOMAXCONN) == -1) {
PLOG_E("listen() for HOST_TO_GUEST on port %u", port);
close(fd);
return -1;
}
}
if (!monitor::addFd(fd, EPOLLIN, nstun::host_callback, ctx)) {
LOG_E("monitor::addFd(host_listener_fd) failed");
close(fd);
return -1;
}
LOG_D("create_host_listener returning fd=%d", fd);
return fd;
}
static bool setup_host_redirect(nstun::Context* ctx, const nstun_rule_t& nr) {
if (nr.direction != NSTUN_DIR_HOST_TO_GUEST || nr.action != NSTUN_ACTION_REDIRECT) {
return true;
}
if (nr.proto != NSTUN_PROTO_TCP && nr.proto != NSTUN_PROTO_UDP) {
LOG_E("HOST_TO_GUEST REDIRECT only supported for TCP/UDP");
return false;
}
if (nr.dport_start == 0) {
LOG_E("HOST_TO_GUEST REDIRECT requires 'dport' to be specified");
return false;
}
if (nr.dport_end < nr.dport_start) {
LOG_E("Invalid port range: %u - %u", nr.dport_start, nr.dport_end);
return false;
}
if (nr.dport_end - nr.dport_start >= 1024) {
LOG_E("Port range too large (%u - %u). Max range is 1024.", nr.dport_start,
nr.dport_end);
return false;
}
uint32_t num_ports = nr.dport_end - nr.dport_start + 1;
if (ctx->num_c_host_listener_rules + num_ports > nstun::NSTUN_MAX_RULES) {
LOG_E("Not enough space for %u host listener rules (current: %zu, max: %zu)",
num_ports, ctx->num_c_host_listener_rules, nstun::NSTUN_MAX_RULES);
return false;
}
int type = (nr.proto == NSTUN_PROTO_TCP) ? SOCK_STREAM : SOCK_DGRAM;
bool is_tcp = (nr.proto == NSTUN_PROTO_TCP);
for (uint32_t port = nr.dport_start; port <= nr.dport_end; port++) {
int fd = -1;
if (nr.is_ipv6) {
struct sockaddr_in6 addr = INIT_SOCKADDR_IN6(AF_INET6);
addr.sin6_port = htons(port);
memcpy(addr.sin6_addr.s6_addr, nr.src_ip6, sizeof(nr.src_ip6));
fd = create_host_listener(AF_INET6, type, (struct sockaddr*)&addr,
sizeof(addr), ctx, is_tcp, port);
} else {
struct sockaddr_in addr = INIT_SOCKADDR_IN(AF_INET);
addr.sin_port = htons(port);
addr.sin_addr.s_addr = nr.src_ip4;
fd = create_host_listener(AF_INET, type, (struct sockaddr*)&addr,
sizeof(addr), ctx, is_tcp, port);
}
if (fd == -1) {
return false;
}
ctx->c_host_listener_rules[ctx->num_c_host_listener_rules].fd = fd;
ctx->c_host_listener_rules[ctx->num_c_host_listener_rules].rule = nr;
ctx->num_c_host_listener_rules++;
LOG_I("Listening on host %s port %u for inbound %s redirection to guest",
nr.is_ipv6 ? "IPv6" : "IPv4", port,
(nr.proto == NSTUN_PROTO_TCP) ? "TCP" : "UDP");
}
return true;
}
static bool parse_rules4(nstun::Context* ctx, nsj_t* nsj) {
for (int i = 0; i < nsj->njc.user_net().rule4_size(); i++) {
const auto& r = nsj->njc.user_net().rule4(i);
nstun_rule_t nr = {};
nstun::RuleParseStatus status = nstun::fill_rule_common(r, &nr);
if (status == nstun::RuleParseStatus::ABORT) return cleanup_and_fail();
if (status == nstun::RuleParseStatus::IGNORE) continue;
if (status == nstun::RuleParseStatus::ABORT) {
return false;
}
if (status == nstun::RuleParseStatus::IGNORE) {
continue;
}
if (r.has_src_ip()) {
parse_ip(r.src_ip(), &nr.src_ip4, &nr.src_mask4);
if (!parse_ip(r.src_ip(), &nr.src_ip4, &nr.src_mask4)) {
return false;
}
}
if (r.has_dst_ip()) {
parse_ip(r.dst_ip(), &nr.dst_ip4, &nr.dst_mask4);
if (!parse_ip(r.dst_ip(), &nr.dst_ip4, &nr.dst_mask4)) {
return false;
}
}
if (r.has_redirect_ip()) {
struct nl_addr* addr;
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET, &addr) == 0) {
if (nl_addr_get_len(addr) == 4) {
memcpy(&nr.redirect_ip4, nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip4));
}
nl_addr_put(addr);
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET, &addr) != 0) {
LOG_E("Failed to parse redirect IP: %s", r.redirect_ip().c_str());
return false;
}
if (nl_addr_get_len(addr) != 4) {
LOG_E("Redirect IP is not IPv4: %s", r.redirect_ip().c_str());
nl_addr_put(addr);
return false;
}
memcpy(&nr.redirect_ip4, nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip4));
nl_addr_put(addr);
}
nr.redirect_port = r.has_redirect_port() ? r.redirect_port() : 0;
ctx->rules.push_back(nr);
if (ctx->c_rules_count >= nstun::NSTUN_MAX_RULES) {
LOG_E("Too many rules (max %zu)", nstun::NSTUN_MAX_RULES);
return false;
}
ctx->c_rules[ctx->c_rules_count++] = nr;
if (nr.direction == NSTUN_DIR_HOST_TO_GUEST && nr.action == NSTUN_ACTION_REDIRECT) {
if (nr.proto != NSTUN_PROTO_TCP && nr.proto != NSTUN_PROTO_UDP) {
LOG_E("HOST_TO_GUEST REDIRECT only supported for TCP/UDP");
return cleanup_and_fail();
}
if (nr.dport_start == 0) {
LOG_E("HOST_TO_GUEST REDIRECT requires 'dport' to be specified");
return cleanup_and_fail();
}
for (uint32_t port = nr.dport_start; port <= nr.dport_end; port++) {
int type = (nr.proto == NSTUN_PROTO_TCP) ? SOCK_STREAM : SOCK_DGRAM;
int fd = socket(AF_INET, type | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd == -1) {
PLOG_E("socket(AF_INET) for HOST_TO_GUEST");
return cleanup_and_fail();
}
int opt = 1;
if (nr.proto == NSTUN_PROTO_TCP) {
if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &opt,
sizeof(opt)) == -1) {
PLOG_W("setsockopt(TCP_NODELAY)");
}
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) ==
-1) {
PLOG_E("setsockopt(SO_REUSEADDR)");
close(fd);
return cleanup_and_fail();
}
struct sockaddr_in addr = INIT_SOCKADDR_IN(AF_INET);
addr.sin_port = htons(port);
addr.sin_addr.s_addr = nr.src_ip4;
if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
PLOG_E("bind() for HOST_TO_GUEST on port %u", port);
close(fd);
return cleanup_and_fail();
}
if (nr.proto == NSTUN_PROTO_TCP) {
if (listen(fd, SOMAXCONN) == -1) {
PLOG_E(
"listen() for HOST_TO_GUEST on port %u", port);
close(fd);
return cleanup_and_fail();
}
}
struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = fd}};
if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) {
PLOG_E("epoll_ctl(EPOLL_CTL_ADD) for HOST_TO_GUEST");
close(fd);
return cleanup_and_fail();
}
ctx->host_listener_fd_to_rule[fd] = nr;
LOG_I(
"Listening on host port %u for inbound %s redirection to guest",
port, (nr.proto == NSTUN_PROTO_TCP) ? "TCP" : "UDP");
}
if (!setup_host_redirect(ctx, nr)) {
return false;
}
}
return true;
}
/* Process IPv6 rules - same NstunRule message, but src_ip4/dst_ip4 parsed as IPv6 */
static bool parse_rules6(nstun::Context* ctx, nsj_t* nsj) {
for (int i = 0; i < nsj->njc.user_net().rule6_size(); i++) {
const auto& r = nsj->njc.user_net().rule6(i);
nstun_rule_t nr = {};
nr.is_ipv6 = true;
nstun::RuleParseStatus status = nstun::fill_rule_common(r, &nr);
if (status == nstun::RuleParseStatus::ABORT) return cleanup_and_fail();
if (status == nstun::RuleParseStatus::IGNORE) continue;
if (status == nstun::RuleParseStatus::ABORT) {
return false;
}
if (status == nstun::RuleParseStatus::IGNORE) {
continue;
}
if (r.has_src_ip()) {
parse_ip6(r.src_ip(), nr.src_ip6, nr.src_mask6);
if (!parse_ip6(r.src_ip(), nr.src_ip6, nr.src_mask6)) {
return false;
}
}
if (r.has_dst_ip()) {
parse_ip6(r.dst_ip(), nr.dst_ip6, nr.dst_mask6);
if (!parse_ip6(r.dst_ip(), nr.dst_ip6, nr.dst_mask6)) {
return false;
}
}
if (r.has_redirect_ip()) {
@@ -407,105 +508,148 @@ bool nstun_init_parent(int sock, nsj_t* nsj) {
nr.action == NSTUN_ACTION_ENCAP_CONNECT) {
/* Proxy is always IPv4 */
struct nl_addr* addr;
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET, &addr) == 0) {
if (nl_addr_get_len(addr) == 4) {
memcpy(&nr.redirect_ip4,
nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip4));
}
nl_addr_put(addr);
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET, &addr) != 0) {
LOG_E("Failed to parse proxy IP: %s",
r.redirect_ip().c_str());
return false;
}
if (nl_addr_get_len(addr) != 4) {
LOG_E("Proxy IP is not IPv4: %s", r.redirect_ip().c_str());
nl_addr_put(addr);
return false;
}
memcpy(&nr.redirect_ip4, nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip4));
nl_addr_put(addr);
} else {
/* REDIRECT: target is IPv6 */
struct nl_addr* addr;
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET6, &addr) == 0) {
if (nl_addr_get_len(addr) == 16) {
memcpy(nr.redirect_ip6,
nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip6));
}
nl_addr_put(addr);
if (nl_addr_parse(r.redirect_ip().c_str(), AF_INET6, &addr) != 0) {
LOG_E("Failed to parse redirect IPv6: %s",
r.redirect_ip().c_str());
return false;
}
if (nl_addr_get_len(addr) != 16) {
LOG_E(
"Redirect IP is not IPv6: %s", r.redirect_ip().c_str());
nl_addr_put(addr);
return false;
}
memcpy(nr.redirect_ip6, nl_addr_get_binary_addr(addr),
sizeof(nr.redirect_ip6));
nl_addr_put(addr);
}
}
nr.redirect_port = r.has_redirect_port() ? r.redirect_port() : 0;
ctx->rules.push_back(nr);
if (ctx->c_rules_count >= nstun::NSTUN_MAX_RULES) {
LOG_E("Too many rules (max %zu)", nstun::NSTUN_MAX_RULES);
return false;
}
ctx->c_rules[ctx->c_rules_count++] = nr;
if (nr.direction == NSTUN_DIR_HOST_TO_GUEST && nr.action == NSTUN_ACTION_REDIRECT) {
if (nr.proto != NSTUN_PROTO_TCP && nr.proto != NSTUN_PROTO_UDP) {
LOG_E("HOST_TO_GUEST REDIRECT only supported for TCP/UDP");
return cleanup_and_fail();
}
if (nr.dport_start == 0) {
LOG_E("HOST_TO_GUEST REDIRECT requires 'dport' to be specified");
return cleanup_and_fail();
}
for (uint32_t port = nr.dport_start; port <= nr.dport_end; port++) {
int type = (nr.proto == NSTUN_PROTO_TCP) ? SOCK_STREAM : SOCK_DGRAM;
int fd = socket(AF_INET6, type | SOCK_NONBLOCK | SOCK_CLOEXEC, 0);
if (fd == -1) {
PLOG_E("socket(AF_INET6) for HOST_TO_GUEST");
return cleanup_and_fail();
}
int opt = 1;
if (nr.proto == NSTUN_PROTO_TCP) {
if (setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &opt,
sizeof(opt)) == -1) {
PLOG_W("setsockopt(TCP_NODELAY)");
}
}
if (setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)) ==
-1) {
PLOG_E("setsockopt(SO_REUSEADDR)");
close(fd);
return cleanup_and_fail();
}
if (setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &opt, sizeof(opt)) ==
-1) {
PLOG_E("setsockopt(IPV6_V6ONLY)");
close(fd);
return cleanup_and_fail();
}
struct sockaddr_in6 addr = INIT_SOCKADDR_IN6(AF_INET6);
addr.sin6_port = htons(port);
memcpy(addr.sin6_addr.s6_addr, nr.src_ip6, sizeof(nr.src_ip6));
if (bind(fd, (struct sockaddr*)&addr, sizeof(addr)) == -1) {
PLOG_E("bind() for HOST_TO_GUEST IPv6 on port %u", port);
close(fd);
return cleanup_and_fail();
}
if (nr.proto == NSTUN_PROTO_TCP) {
if (listen(fd, SOMAXCONN) == -1) {
PLOG_E("listen() for HOST_TO_GUEST IPv6 on port %u",
port);
close(fd);
return cleanup_and_fail();
}
}
struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = fd}};
if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, fd, &ev) == -1) {
PLOG_E("epoll_ctl(EPOLL_CTL_ADD) for HOST_TO_GUEST IPv6");
close(fd);
return cleanup_and_fail();
}
ctx->host_listener_fd_to_rule[fd] = nr;
LOG_I("Listening on host IPv6 port %u for inbound %s redirection "
"to guest",
port, (nr.proto == NSTUN_PROTO_TCP) ? "TCP" : "UDP");
}
if (!setup_host_redirect(ctx, nr)) {
return false;
}
}
/* Spawn network loop thread */
std::thread t(nstun::networkLoop, ctx.release());
t.detach();
return true;
}
static bool setup_ip4(const std::string& config_ip, const char* default_ip, uint32_t* out_ip) {
if (!config_ip.empty()) {
return assign_ip(config_ip, out_ip);
}
if (inet_pton(AF_INET, default_ip, out_ip) != 1) {
LOG_E("Failed to parse default IP4: %s", default_ip);
return false;
}
return true;
}
static bool setup_ip6(const std::string& config_ip, const char* default_ip, uint8_t* out_ip6) {
if (!config_ip.empty()) {
if (inet_pton(AF_INET6, config_ip.c_str(), out_ip6) != 1) {
LOG_E("Cannot convert '%s' into an IPv6 address", config_ip.c_str());
return false;
}
return true;
}
if (inet_pton(AF_INET6, default_ip, out_ip6) != 1) {
LOG_E("Failed to parse default IP6: %s", default_ip);
return false;
}
return true;
}
bool nstun_init_parent(int tap_fd, nsj_t* nsj, pid_t pid) {
LOG_D("nstun initialized successfully, tap_fd=%d", tap_fd);
nstun::Context* ctx = new (std::nothrow) nstun::Context();
if (!ctx) {
LOG_E("Failed to allocate Context");
return false;
}
LOG_D("nstun_init_parent: allocated ctx=%p", ctx);
ctx->tap_fd = tap_fd;
bool success = false;
defer {
if (!success) {
/* Caller (monitorThread) closes tap_fd on failure, so detach it to prevent
* double-close */
ctx->tap_fd = -1;
context_cleanup(ctx);
delete ctx;
}
};
if (!setup_ip4(nsj->njc.user_net().ip4(), "192.168.0.2", &ctx->guest_ip4)) {
return false;
}
if (!setup_ip4(nsj->njc.user_net().gw4(), "192.168.0.1", &ctx->host_ip4)) {
return false;
}
if (!setup_ip6(nsj->njc.user_net().ip6(), "fd00::2", ctx->guest_ip6)) {
return false;
}
if (!setup_ip6(nsj->njc.user_net().gw6(), "fd00::1", ctx->host_ip6)) {
return false;
}
if (!parse_rules4(ctx, nsj)) {
return false;
}
if (!parse_rules6(ctx, nsj)) {
return false;
}
/* Register with monitor loop */
if (!monitor::addFd(ctx->tap_fd, EPOLLIN, tapCb, ctx)) {
LOG_E("monitor::addFd(tap_fd) failed");
return false;
}
for (int i = 0; i < nstun::VLEN; ++i) {
ctx->recvmmsg_iovecs[i].iov_base = ctx->recvmmsg_bufs[i];
ctx->recvmmsg_iovecs[i].iov_len = sizeof(ctx->recvmmsg_bufs[i]);
ctx->recvmmsg_msgs[i].msg_hdr.msg_iov = &ctx->recvmmsg_iovecs[i];
ctx->recvmmsg_msgs[i].msg_hdr.msg_iovlen = 1;
ctx->recvmmsg_msgs[i].msg_hdr.msg_name = &ctx->recvmmsg_addrs[i];
ctx->recvmmsg_msgs[i].msg_hdr.msg_control = nullptr;
ctx->recvmmsg_msgs[i].msg_hdr.msg_controllen = 0;
}
ctx->recvmmsg_initialized = true;
tls_nstun_ctx = ctx;
success = true;
return true;
}
void nstun_destroy_parent() {
if (tls_nstun_ctx) {
context_cleanup(tls_nstun_ctx);
delete tls_nstun_ctx;
tls_nstun_ctx = nullptr;
}
}

16
nstun/nstun.h
View File

@@ -5,9 +5,9 @@
#include <stdint.h>
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
#include <memory>
#include "monitor.h"
typedef enum {
NSTUN_ACTION_DROP,
@@ -50,11 +50,9 @@ typedef struct {
} nstun_rule_t;
struct nsj_t;
bool nstun_init_child(int sock, struct nsj_t* nsj);
bool nstun_init_parent(int sock, struct nsj_t* nsj);
#ifdef __cplusplus
}
#endif
bool nstun_init_child(int ipc_fd, struct nsj_t* nsj);
bool nstun_init_parent(int tap_fd, struct nsj_t* nsj, pid_t pid);
void nstun_destroy_parent();
void nstun_periodic();
#endif /* NSTUN_H_ */

161
nstun/policy.cc
View File

@@ -1,5 +1,6 @@
#include "policy.h"
#include <netinet/in.h>
#include <string.h>
#include "core.h"
@@ -14,20 +15,40 @@ namespace nstun {
RuleResult evaluate_rules4(Context* ctx, nstun_direction_t dir, nstun_proto_t proto,
uint32_t src_ip4, uint32_t dst_ip4, uint16_t sport, uint16_t dport) {
for (const auto& r : ctx->rules) {
if (r.is_ipv6) continue;
if (r.direction != dir) continue;
if (r.proto != NSTUN_PROTO_ANY && r.proto != proto) continue;
if (r.src_ip4 != 0 && (src_ip4 & r.src_mask4) != (r.src_ip4 & r.src_mask4))
LOG_D("evaluate_rules4: rules count=%zu", ctx->c_rules_count);
for (size_t i = 0; i < ctx->c_rules_count; ++i) {
const auto& r = ctx->c_rules[i];
if (r.is_ipv6) {
continue;
if (r.dst_ip4 != 0 && (dst_ip4 & r.dst_mask4) != (r.dst_ip4 & r.dst_mask4))
}
if (r.direction != dir) {
continue;
}
if (r.proto != NSTUN_PROTO_ANY && r.proto != proto) {
continue;
}
if (r.sport_start != 0 && (sport < r.sport_start || sport > r.sport_end)) continue;
if (r.dport_start != 0 && (dport < r.dport_start || dport > r.dport_end)) continue;
if (r.src_mask4 != 0 && (src_ip4 & r.src_mask4) != (r.src_ip4 & r.src_mask4)) {
continue;
}
if (r.dst_mask4 != 0 && (dst_ip4 & r.dst_mask4) != (r.dst_ip4 & r.dst_mask4)) {
continue;
}
RuleResult res = {r.action, 0, 0, false, {}};
if (r.sport_start != 0 && (sport < r.sport_start || sport > r.sport_end)) {
continue;
}
if (r.dport_start != 0 && (dport < r.dport_start || dport > r.dport_end)) {
continue;
}
RuleResult res = {
.action = r.action,
.redirect_ip4 = 0,
.redirect_port = 0,
.has_redirect_ip6 = false,
.redirect_ip6 = {},
};
if (r.action == NSTUN_ACTION_REDIRECT || r.action == NSTUN_ACTION_ENCAP_SOCKS5 ||
r.action == NSTUN_ACTION_ENCAP_CONNECT) {
res.redirect_ip4 = r.redirect_ip4;
@@ -35,39 +56,62 @@ RuleResult evaluate_rules4(Context* ctx, nstun_direction_t dir, nstun_proto_t pr
}
return res;
}
return {NSTUN_ACTION_ALLOW, 0, 0, false, {}}; /* Default allow */
return {
.action = NSTUN_ACTION_ALLOW,
.redirect_ip4 = 0,
.redirect_port = 0,
.has_redirect_ip6 = false,
.redirect_ip6 = {},
};
}
static bool ip6_masked_eq(const uint8_t* a, const uint8_t* b, const uint8_t* mask) {
for (int i = 0; i < 16; i++) {
if ((a[i] & mask[i]) != (b[i] & mask[i])) return false;
}
return true;
}
static bool ip6_is_zero(const uint8_t* addr) {
for (int i = 0; i < 16; i++) {
if (addr[i] != 0) return false;
if ((a[i] & mask[i]) != (b[i] & mask[i])) {
return false;
}
}
return true;
}
RuleResult evaluate_rules6(Context* ctx, nstun_direction_t dir, nstun_proto_t proto,
const uint8_t* src_ip6, const uint8_t* dst_ip6, uint16_t sport, uint16_t dport) {
for (const auto& r : ctx->rules) {
if (!r.is_ipv6) continue;
if (r.direction != dir) continue;
if (r.proto != NSTUN_PROTO_ANY && r.proto != proto) continue;
if (!ip6_is_zero(r.src_ip6) && !ip6_masked_eq(src_ip6, r.src_ip6, r.src_mask6))
static const uint8_t kZeroIp6[16] = {0};
for (size_t i = 0; i < ctx->c_rules_count; ++i) {
const auto& r = ctx->c_rules[i];
if (!r.is_ipv6) {
continue;
if (!ip6_is_zero(r.dst_ip6) && !ip6_masked_eq(dst_ip6, r.dst_ip6, r.dst_mask6))
}
if (r.direction != dir) {
continue;
}
if (r.proto != NSTUN_PROTO_ANY && r.proto != proto) {
continue;
}
if (r.sport_start != 0 && (sport < r.sport_start || sport > r.sport_end)) continue;
if (r.dport_start != 0 && (dport < r.dport_start || dport > r.dport_end)) continue;
if (memcmp(r.src_mask6, kZeroIp6, 16) != 0 &&
!ip6_masked_eq(src_ip6, r.src_ip6, r.src_mask6)) {
continue;
}
if (memcmp(r.dst_mask6, kZeroIp6, 16) != 0 &&
!ip6_masked_eq(dst_ip6, r.dst_ip6, r.dst_mask6)) {
continue;
}
RuleResult res = {r.action, 0, 0, false, {}};
if (r.sport_start != 0 && (sport < r.sport_start || sport > r.sport_end)) {
continue;
}
if (r.dport_start != 0 && (dport < r.dport_start || dport > r.dport_end)) {
continue;
}
RuleResult res = {
.action = r.action,
.redirect_ip4 = 0,
.redirect_port = 0,
.has_redirect_ip6 = false,
.redirect_ip6 = {},
};
if (r.action == NSTUN_ACTION_REDIRECT) {
res.has_redirect_ip6 = true;
memcpy(res.redirect_ip6, r.redirect_ip6, sizeof(res.redirect_ip6));
@@ -80,11 +124,17 @@ RuleResult evaluate_rules6(Context* ctx, nstun_direction_t dir, nstun_proto_t pr
}
return res;
}
return {NSTUN_ACTION_ALLOW, 0, 0, false, {}}; /* Default allow */
return {
.action = NSTUN_ACTION_ALLOW,
.redirect_ip4 = 0,
.redirect_port = 0,
.has_redirect_ip6 = false,
.redirect_ip6 = {},
};
}
template <typename RuleMsg>
RuleParseStatus fill_rule_common(const RuleMsg& r, nstun_rule_t* nr) {
RuleParseStatus fill_rule_common(
const nsjail::NsJailConfig_UserNet_NstunRule& r, nstun_rule_t* nr) {
if ((r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_SOCKS5 ||
r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_CONNECT) &&
r.proto() == nsjail::NsJailConfig_UserNet_NstunRule_Protocol_ICMP) {
@@ -98,30 +148,42 @@ RuleParseStatus fill_rule_common(const RuleMsg& r, nstun_rule_t* nr) {
nr->direction = NSTUN_DIR_GUEST_TO_HOST;
}
if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_DROP) {
switch (r.action()) {
case nsjail::NsJailConfig_UserNet_NstunRule_Action_DROP:
nr->action = NSTUN_ACTION_DROP;
} else if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_REJECT) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Action_REJECT:
nr->action = NSTUN_ACTION_REJECT;
} else if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_ALLOW) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Action_ALLOW:
nr->action = NSTUN_ACTION_ALLOW;
} else if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_REDIRECT) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Action_REDIRECT:
nr->action = NSTUN_ACTION_REDIRECT;
} else if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_SOCKS5) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_SOCKS5:
nr->action = NSTUN_ACTION_ENCAP_SOCKS5;
} else if (r.action() == nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_CONNECT) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Action_ENCAP_CONNECT:
nr->action = NSTUN_ACTION_ENCAP_CONNECT;
} else {
break;
default:
return RuleParseStatus::IGNORE;
}
if (r.proto() == nsjail::NsJailConfig_UserNet_NstunRule_Protocol_TCP) {
switch (r.proto()) {
case nsjail::NsJailConfig_UserNet_NstunRule_Protocol_TCP:
nr->proto = NSTUN_PROTO_TCP;
} else if (r.proto() == nsjail::NsJailConfig_UserNet_NstunRule_Protocol_UDP) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Protocol_UDP:
nr->proto = NSTUN_PROTO_UDP;
} else if (r.proto() == nsjail::NsJailConfig_UserNet_NstunRule_Protocol_ICMP) {
break;
case nsjail::NsJailConfig_UserNet_NstunRule_Protocol_ICMP:
nr->proto = NSTUN_PROTO_ICMP;
} else {
break;
default:
nr->proto = NSTUN_PROTO_ANY;
break;
}
nr->sport_start = r.has_sport() ? r.sport() : 0;
@@ -130,11 +192,16 @@ RuleParseStatus fill_rule_common(const RuleMsg& r, nstun_rule_t* nr) {
nr->dport_start = r.has_dport() ? r.dport() : 0;
nr->dport_end = r.has_dport_end() ? r.dport_end() : nr->dport_start;
if (nr->sport_end < nr->sport_start) {
LOG_E("Invalid source port range: %u - %u", nr->sport_start, nr->sport_end);
return RuleParseStatus::ABORT;
}
if (nr->dport_end < nr->dport_start) {
LOG_E("Invalid destination port range: %u - %u", nr->dport_start, nr->dport_end);
return RuleParseStatus::ABORT;
}
return RuleParseStatus::OK;
}
/* Explicit template instantiation for the protobuf rule message type */
template RuleParseStatus fill_rule_common<nsjail::NsJailConfig_UserNet_NstunRule>(
const nsjail::NsJailConfig_UserNet_NstunRule& r, nstun_rule_t* nr);
} /* namespace nstun */

4
nstun/policy.h
View File

@@ -1,6 +1,7 @@
#ifndef NSTUN_POLICY_H_
#define NSTUN_POLICY_H_
#include "config.pb.h"
#include "core.h"
#include "nstun.h"
@@ -18,8 +19,7 @@ RuleResult evaluate_rules4(Context* ctx, nstun_direction_t dir, nstun_proto_t pr
RuleResult evaluate_rules6(Context* ctx, nstun_direction_t dir, nstun_proto_t proto,
const uint8_t* src_ip6, const uint8_t* dst_ip6, uint16_t sport, uint16_t dport);
template <typename RuleMsg>
RuleParseStatus fill_rule_common(const RuleMsg& r, nstun_rule_t* nr);
RuleParseStatus fill_rule_common(const nsjail::NsJailConfig_UserNet_NstunRule& r, nstun_rule_t* nr);
} /* namespace nstun */

2200
nstun/tcp.cc
View File

File diff suppressed because it is too large Load Diff

78
nstun/tcp.h
View File

@@ -1,83 +1,23 @@
#ifndef NSTUN_TCP_H_
#define NSTUN_TCP_H_
#include <deque>
#include <span>
#include <vector>
#include "core.h"
namespace nstun {
enum class TcpState {
SYN_SENT, /* Host connecting to destination */
SOCKS5_INIT, /* Sent SOCKS5 greeting, awaiting auth reply */
SOCKS5_CONNECTING, /* Sent SOCKS5 CONNECT request, awaiting response */
HTTP_CONNECT_WAIT, /* Sent HTTP CONNECT, awaiting proxy 200 reply */
ESTABLISHED,
FIN_WAIT_1,
FIN_WAIT_2,
CLOSING,
TIME_WAIT,
CLOSE_WAIT,
};
void handle_host_tcp_event(Context* ctx, TcpFlow* flow, int fd, uint32_t events);
void tcp_periodic_check(Context* ctx, TcpFlow* flow, time_t now);
bool is_stale_tcp(const TcpFlow* flow, time_t now);
struct TcpFlow : public Flow {
int host_fd = -1;
union {
FlowKey4 key4;
FlowKey6 key6;
};
TcpState state = TcpState::SYN_SENT;
ProxyMode proxy_mode = ProxyMode::NONE;
bool host_eof = false;
bool guest_eof = false;
bool fin_sent = false;
bool syn_acked = false;
bool fin_acked = false;
uint32_t seq_to_guest = 0;
uint32_t ack_from_guest = 0;
uint32_t seq_from_guest = 0;
uint32_t ack_to_guest = 0;
/* Buffer for data from host to guest (not yet ACKed) */
/* In a real TCP stack, this would handle retransmissions. */
/* Here, we just queue it to send. */
std::vector<uint8_t> tx_buffer;
size_t tx_acked_offset = 0;
/* Buffer for accumulating proxy handshake responses (SOCKS5/HTTP CONNECT) */
std::vector<uint8_t> proxy_rx_buffer;
/* Buffer for data from guest to host to avoid dropping packets on EAGAIN */
std::vector<uint8_t> rx_buffer;
size_t rx_sent_offset = 0;
bool epoll_out_registered = false;
bool epoll_in_disabled = false;
bool inbound = false; /* true if flow is HOST_TO_GUEST */
~TcpFlow() override {
if (host_fd != -1) ::close(host_fd);
}
void handle_host_event(Context* ctx, int fd, uint32_t events) override;
void periodic_check(Context* ctx, time_t now) override;
bool is_stale(time_t now) const override;
void destroy(Context* ctx) override;
};
void tcp_send_packet4(
Context* ctx, TcpFlow* flow, uint8_t flags, const uint8_t* data = nullptr, size_t len = 0);
void tcp_send_packet6(
Context* ctx, TcpFlow* flow, uint8_t flags, const uint8_t* data = nullptr, size_t len = 0);
bool tcp_send_packet4(Context* ctx, const TcpFlow* flow, uint8_t flags,
const uint8_t* data = nullptr, size_t len = 0);
bool tcp_send_packet6(Context* ctx, const TcpFlow* flow, uint8_t flags,
const uint8_t* data = nullptr, size_t len = 0);
void tcp_destroy_flow(Context* ctx, TcpFlow* flow);
void push_to_guest(Context* ctx, TcpFlow* flow);
void handle_tcp4(Context* ctx, const ip4_hdr* ip, std::span<const uint8_t> payload);
void handle_tcp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payload);
void handle_tcp4(Context* ctx, const ip4_hdr* ip, const uint8_t* data, size_t len);
void handle_tcp6(Context* ctx, const ip6_hdr* ip, const uint8_t* data, size_t len);
void handle_host_tcp(Context* ctx, TcpFlow* flow, uint32_t events);
void handle_host_tcp_accept(Context* ctx, int listen_fd, const nstun_rule_t& rule);

10
nstun/tun.cc
View File

@@ -6,8 +6,6 @@
#include <sys/uio.h>
#include <unistd.h>
#include <span>
#include "core.h"
#include "icmp.h"
#include "ip.h"
@@ -23,9 +21,9 @@ bool send_to_guest_v(
}
struct iovec iov[2];
iov[0].iov_base = (void*)header;
iov[0].iov_base = const_cast<void*>(header);
iov[0].iov_len = header_len;
iov[1].iov_base = (void*)payload;
iov[1].iov_base = const_cast<void*>(payload);
iov[1].iov_len = payload_len;
size_t total_len = header_len + payload_len;
@@ -54,10 +52,10 @@ void handle_tun_frame(Context* ctx, const uint8_t* buf, size_t len) {
switch (version) {
case 4:
handle_ip4(ctx, std::span(buf, len));
handle_ip4(ctx, buf, len);
break;
case 6:
handle_ip6(ctx, std::span(buf, len));
handle_ip6(ctx, buf, len);
break;
default:
LOG_D("Unknown IP version: %u", version);

1478
nstun/udp.cc
View File

File diff suppressed because it is too large Load Diff

9
nstun/udp.h
View File

@@ -4,15 +4,16 @@
#include <stddef.h>
#include <stdint.h>
#include <span>
#include "core.h"
namespace nstun {
void handle_udp4(Context* ctx, const ip4_hdr* ip, std::span<const uint8_t> payload);
void handle_udp6(Context* ctx, const ip6_hdr* ip, std::span<const uint8_t> payload);
void handle_udp4(Context* ctx, const ip4_hdr* ip, const uint8_t* data, size_t len);
void handle_udp6(Context* ctx, const ip6_hdr* ip, const uint8_t* data, size_t len);
void handle_host_udp_accept(Context* ctx, int listen_fd, const nstun_rule_t& rule);
void handle_host_udp_event(Context* ctx, UdpFlow* flow, int fd, uint32_t events);
bool is_stale_udp(const UdpFlow* flow, time_t now);
void udp_destroy_flow(Context* ctx, UdpFlow* flow);
} // namespace nstun

2
pid.cc
View File

@@ -48,7 +48,7 @@ bool initNs(nsj_t* nsj) {
* first clone/fork will work, and the rest will fail with ENOMEM (see 'man pid_namespaces'
* for details on this behavior)
*/
pid_t pid = subproc::cloneProc(CLONE_FS, 0);
pid_t pid = subproc::cloneProcNoPidfd(CLONE_FS, 0);
if (pid == -1) {
PLOG_E("Couldn't create a dummy init process");
return false;

35
sandbox.cc
View File

@@ -36,28 +36,14 @@ extern "C" {
#include "kafel.h"
}
#include "logs.h"
#include "missing_defs.h"
#include "monitor.h"
#include "unotify/syscall_defs.h"
#include "util.h"
namespace sandbox {
#ifndef PR_SET_NO_NEW_PRIVS /* in prctl.h since Linux 3.5 */
#define PR_SET_NO_NEW_PRIVS 38
#endif /* PR_SET_NO_NEW_PRIVS */
#ifndef SECCOMP_FILTER_FLAG_TSYNC
#define SECCOMP_FILTER_FLAG_TSYNC (1UL << 0)
#endif /* SECCOMP_FILTER_FLAG_TSYNC */
#ifndef SECCOMP_FILTER_FLAG_LOG
#define SECCOMP_FILTER_FLAG_LOG (1UL << 1)
#endif /* SECCOMP_FILTER_FLAG_LOG */
#ifndef SECCOMP_FILTER_FLAG_NEW_LISTENER
#define SECCOMP_FILTER_FLAG_NEW_LISTENER (1UL << 3)
#endif /* SECCOMP_FILTER_FLAG_NEW_LISTENER */
bool installUnotifyFilter(nsj_t* nsj, int pipefd) {
bool installUnotifyFilter(nsj_t* nsj, int ipc_fd) {
if (!nsj->njc.seccomp_unotify()) {
return true;
}
@@ -85,8 +71,8 @@ bool installUnotifyFilter(nsj_t* nsj, int pipefd) {
return false;
}
if (!util::sendFd(pipefd, unotif_fd)) {
PLOG_E("sendFd(unotif_fd) to parent failed");
if (!util::sendMsg(ipc_fd, monitor::MSG_TAG_UNOTIFY, unotif_fd)) {
PLOG_E("sendMsg(unotif_fd) to parent failed");
close(unotif_fd);
return false;
}
@@ -127,9 +113,9 @@ static bool prepareAndCommit(nsj_t* nsj) {
return true;
}
bool applyPolicy(nsj_t* nsj, int pipefd) {
if (pipefd != -1 && nsj->njc.seccomp_unotify()) {
if (!installUnotifyFilter(nsj, pipefd)) {
bool applyPolicy(nsj_t* nsj, int ipc_fd) {
if (ipc_fd != -1 && nsj->njc.seccomp_unotify()) {
if (!installUnotifyFilter(nsj, ipc_fd)) {
return false;
}
}
@@ -171,8 +157,9 @@ bool preparePolicy(nsj_t* nsj) {
kafel_ctxt_t ctxt = kafel_ctxt_create();
std::string combined_seccomp_policy;
FILE* f = nullptr;
if (!nsj->njc.seccomp_policy_file().empty()) {
FILE* f = fopen(nsj->njc.seccomp_policy_file().c_str(), "r");
f = fopen(nsj->njc.seccomp_policy_file().c_str(), "re");
if (!f) {
PLOG_W("Couldn't open the kafel seccomp policy file '%s'",
nsj->njc.seccomp_policy_file().c_str());
@@ -197,9 +184,11 @@ bool preparePolicy(nsj_t* nsj) {
if (kafel_compile(ctxt, &nsj->seccomp_fprog) != 0) {
LOG_E("Could not compile policy: %s", kafel_error_msg(ctxt));
kafel_ctxt_destroy(&ctxt);
if (f) fclose(f);
return false;
}
kafel_ctxt_destroy(&ctxt);
if (f) fclose(f);
return true;
}

4
sandbox.h
View File

@@ -28,8 +28,8 @@
namespace sandbox {
bool installUnotifyFilter(nsj_t* nsj, int pipefd);
bool applyPolicy(nsj_t* nsj, int pipefd);
bool installUnotifyFilter(nsj_t* nsj, int ipc_fd);
bool applyPolicy(nsj_t* nsj, int ipc_fd);
bool preparePolicy(nsj_t* nsj);
void closePolicy(nsj_t* nsj);

375
sockproxy/sockproxy.cc Normal file
View File

@@ -0,0 +1,375 @@
/*
* nsjail - socket proxy for listen mode (-Ml)
* -----------------------------------------
*
* Proxies data between an external TCP connection (sock_fd) and the
* child process's stdin/stdout pipes using splice(2) for zero-copy
* transfer.
*
* Two independent half-duplex channels:
* sock_fd -> pipe_in (network to child stdin)
* pipe_out -> sock_fd (child stdout to network)
*
* Each channel drains independently with half-close semantics.
* When both channels are done, the proxy tears itself down and
* signals the monitor to stop the event loop.
*/
#include "sockproxy.h"
#include <errno.h>
#include <fcntl.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <sys/socket.h>
#include <unistd.h>
#include "logs.h"
#include "monitor.h"
#include "util.h"
namespace sockproxy {
static constexpr size_t kSpliceChunkSize = 65536;
static constexpr int kMaxSpliceLoops = 16;
/* --- connection state ---------------------------------- */
struct channel_t {
int pipe_fd = -1;
bool blocked = false;
bool registered = false;
};
struct conn_t {
int sock_fd = -1;
bool sock_registered = false;
channel_t sock_to_pipe;
channel_t pipe_to_sock;
on_close_cb_t close_cb = nullptr;
void* cb_data = nullptr;
};
static thread_local conn_t current_conn;
/* --- helpers ------------------------------------------- */
/*
* Remove an FD from epoll, close it, and invalidate the slot.
* Safe to call with *fd == -1 (no-op).
*/
static void closeAndUnregister(channel_t* chan) {
if (chan->pipe_fd >= 0) {
if (chan->registered) {
monitor::removeFd(chan->pipe_fd);
chan->registered = false;
}
close(chan->pipe_fd);
chan->pipe_fd = -1;
}
}
/*
* Tear down an entire connection: unregister all FDs from epoll,
* close everything, and clear the state.
*/
static void teardownConn(conn_t* conn) {
if (!conn) {
return;
}
closeAndUnregister(&conn->sock_to_pipe);
closeAndUnregister(&conn->pipe_to_sock);
if (conn->sock_fd >= 0) {
if (conn->sock_registered) {
monitor::removeFd(conn->sock_fd);
conn->sock_registered = false;
}
close(conn->sock_fd);
conn->sock_fd = -1;
}
}
/*
* Tear down connection and notify monitor.
*/
static void conclude(conn_t* conn) {
teardownConn(conn);
if (conn->close_cb) {
conn->close_cb(conn->cb_data);
} else {
monitor::stop();
}
}
/* --- splice pump --------------------------------------- */
enum class PumpResult {
kMoved, /* data transferred, loop again */
kEof, /* EOF or hard error -- close channel */
kBlocked, /* dest full, need EPOLLOUT on dest */
kDrained, /* source empty, need EPOLLIN on src */
};
/*
* One non-blocking splice(src -> dst).
* Exactly one of src/dst must be a pipe end (splice(2) requirement).
* On EAGAIN, FIONREAD on check_fd distinguishes "source empty"
* from "destination full".
*/
static PumpResult splicePump(int src, int dst, int check_fd) {
ssize_t r = TEMP_FAILURE_RETRY(
splice(src, nullptr, dst, nullptr, kSpliceChunkSize, SPLICE_F_NONBLOCK));
if (r > 0) {
return PumpResult::kMoved;
}
if (r == 0) {
return PumpResult::kEof;
}
if (r == -1 && errno != EAGAIN && errno != EWOULDBLOCK) {
PLOG_D("splice(src=%d, dst=%d) error", src, dst);
return PumpResult::kEof;
}
/* EAGAIN: source empty or dest full? */
int n = 0;
if (ioctl(check_fd, FIONREAD, &n) == -1) {
PLOG_W("ioctl(FIONREAD, fd=%d)", check_fd);
return PumpResult::kEof;
}
return (n == 0) ? PumpResult::kDrained : PumpResult::kBlocked;
}
/*
* Drain one direction until it can't proceed.
* On EOF/error, *pipe_fd is removed from epoll and closed.
* Returns false if the channel hit EOF (*pipe_fd is now -1).
*/
static bool drainChannel(int src, int dst, int check_fd, channel_t* chan) {
int loops = 0;
while (!chan->blocked && chan->pipe_fd >= 0) {
switch (splicePump(src, dst, check_fd)) {
case PumpResult::kMoved:
if (++loops > kMaxSpliceLoops) {
return true; /* Yield to event loop for fairness */
}
continue;
case PumpResult::kEof:
closeAndUnregister(chan);
return false;
case PumpResult::kBlocked:
chan->blocked = true;
return true;
case PumpResult::kDrained:
return true;
}
}
return (chan->pipe_fd >= 0);
}
static void proxyPumpCb(int fd, uint32_t events, void* data);
static void updateFdMask(int fd, uint32_t events, bool* registered, conn_t* conn) {
if (fd < 0) {
return;
}
if (events != 0) {
if (!*registered) {
if (monitor::addFd(fd, events, proxyPumpCb, conn)) {
*registered = true;
}
} else {
monitor::modFd(fd, events);
}
} else {
if (*registered) {
monitor::removeFd(fd);
*registered = false;
}
}
}
/*
* Recompute epoll event masks for all three FDs.
*
* Per-direction logic:
* not blocked -> watch source for EPOLLIN (data available)
* blocked -> watch dest for EPOLLOUT (space available)
*/
static void updateMasks(conn_t* conn) {
uint32_t sock_ev = 0;
uint32_t pipe_in_ev = 0;
uint32_t pipe_out_ev = 0;
if (conn->sock_fd >= 0 && conn->sock_to_pipe.pipe_fd >= 0) {
sock_ev |= EPOLLRDHUP;
}
/* sock -> pipe_in */
if (!conn->sock_to_pipe.blocked && conn->sock_to_pipe.pipe_fd >= 0) {
sock_ev |= EPOLLIN;
}
if (conn->sock_to_pipe.blocked && conn->sock_fd >= 0) {
pipe_in_ev |= EPOLLOUT;
}
/* pipe_out -> sock */
if (!conn->pipe_to_sock.blocked && conn->pipe_to_sock.pipe_fd >= 0) {
pipe_out_ev |= EPOLLIN;
}
if (conn->pipe_to_sock.blocked && conn->pipe_to_sock.pipe_fd >= 0) {
sock_ev |= EPOLLOUT;
}
updateFdMask(conn->sock_fd, sock_ev, &conn->sock_registered, conn);
updateFdMask(conn->sock_to_pipe.pipe_fd, pipe_in_ev, &conn->sock_to_pipe.registered, conn);
updateFdMask(conn->pipe_to_sock.pipe_fd, pipe_out_ev, &conn->pipe_to_sock.registered, conn);
}
/* --- epoll callback ------------------------------------ */
static void proxyPumpCb(int fd, uint32_t events, void* data) {
conn_t* conn = static_cast<conn_t*>(data);
LOG_D("proxyPumpCb fd=%d events=0x%x", fd, events);
if (events & EPOLLERR) {
LOG_D("EPOLLERR on fd=%d, tearing down connection", fd);
conclude(conn);
return;
}
bool handled = false;
/* -- Direction 1: sock_fd -> pipe_in (network -> child stdin) -- */
if (conn->sock_to_pipe.pipe_fd >= 0 && conn->sock_fd >= 0) {
bool ready = false;
if (fd == conn->sock_fd &&
(events & (EPOLLIN | EPOLLRDHUP | EPOLLHUP | EPOLLERR))) {
ready = true;
}
if (fd == conn->sock_to_pipe.pipe_fd &&
(events & (EPOLLOUT | EPOLLHUP | EPOLLERR))) {
ready = true;
}
if (ready) {
conn->sock_to_pipe.blocked = false;
if (!drainChannel(conn->sock_fd, conn->sock_to_pipe.pipe_fd, conn->sock_fd,
&conn->sock_to_pipe)) {
LOG_D("sock->pipe EOF (client half-closed), closing child stdin");
}
handled = true;
}
}
/* -- Direction 2: pipe_out -> sock_fd (child stdout -> network) -- */
if (conn->pipe_to_sock.pipe_fd >= 0 && conn->sock_fd >= 0) {
bool ready = false;
if (fd == conn->pipe_to_sock.pipe_fd &&
(events & (EPOLLIN | EPOLLRDHUP | EPOLLHUP | EPOLLERR))) {
ready = true;
}
if (fd == conn->sock_fd && (events & EPOLLOUT)) {
ready = true;
}
if (ready) {
conn->pipe_to_sock.blocked = false;
if (!drainChannel(conn->pipe_to_sock.pipe_fd, conn->sock_fd,
conn->pipe_to_sock.pipe_fd, &conn->pipe_to_sock)) {
LOG_D("pipe->sock EOF, half-closing socket write side");
shutdown(conn->sock_fd, SHUT_WR);
}
handled = true;
}
}
/*
* Socket broken (EPOLLHUP/EPOLLERR): stop feeding the child.
* pipe_out may still have buffered data but splice will fail
* writing to a dead socket, so both channels converge to -1.
*/
if (fd == conn->sock_fd && (events & (EPOLLHUP | EPOLLERR))) {
LOG_D("Socket error/hup on sock_fd=%d, tearing down", fd);
conclude(conn);
return;
}
/* -- Both done: tear down proxy and signal monitor to stop -- */
if (conn->sock_to_pipe.pipe_fd == -1 && conn->pipe_to_sock.pipe_fd == -1) {
LOG_D("Proxy fully drained, tearing down");
conclude(conn);
} else if (handled) {
updateMasks(conn);
}
}
/* --- public API ---------------------------------------- */
bool start(int* connfd, int* pipe_in, int* pipe_out, on_close_cb_t cb, void* data) {
conn_t* conn = &current_conn;
conn->sock_fd = *connfd;
conn->sock_to_pipe.pipe_fd = *pipe_in;
conn->pipe_to_sock.pipe_fd = *pipe_out;
conn->close_cb = cb;
conn->cb_data = data;
/* Belt-and-suspenders: make sock_fd non-blocking in addition to
* SPLICE_F_NONBLOCK, so FIONREAD probes and future read()/write()
* calls never block the event loop. */
if (!util::setNonBlock(conn->sock_fd)) {
conn->sock_fd = -1;
conn->sock_to_pipe.pipe_fd = -1;
conn->pipe_to_sock.pipe_fd = -1;
return false;
}
if (!monitor::addFd(conn->sock_fd, EPOLLRDHUP, proxyPumpCb, conn)) {
PLOG_E("addFd(sock_fd=%d)", conn->sock_fd);
conn->sock_fd = -1;
conn->sock_to_pipe.pipe_fd = -1;
conn->pipe_to_sock.pipe_fd = -1;
return false;
}
conn->sock_registered = true;
if (!monitor::addFd(conn->sock_to_pipe.pipe_fd, EPOLLRDHUP, proxyPumpCb, conn)) {
PLOG_E("addFd(pipe_in=%d)", conn->sock_to_pipe.pipe_fd);
monitor::removeFd(conn->sock_fd);
conn->sock_registered = false;
conn->sock_fd = -1;
conn->sock_to_pipe.pipe_fd = -1;
conn->pipe_to_sock.pipe_fd = -1;
return false;
}
conn->sock_to_pipe.registered = true;
if (!monitor::addFd(conn->pipe_to_sock.pipe_fd, EPOLLRDHUP, proxyPumpCb, conn)) {
PLOG_E("addFd(pipe_out=%d)", conn->pipe_to_sock.pipe_fd);
monitor::removeFd(conn->sock_to_pipe.pipe_fd);
conn->sock_to_pipe.registered = false;
monitor::removeFd(conn->sock_fd);
conn->sock_registered = false;
conn->sock_fd = -1;
conn->sock_to_pipe.pipe_fd = -1;
conn->pipe_to_sock.pipe_fd = -1;
return false;
}
conn->pipe_to_sock.registered = true;
/* Success: proxy owns the FDs now, invalidate caller's copies */
*connfd = -1;
*pipe_in = -1;
*pipe_out = -1;
updateMasks(conn);
return true;
}
void stop() {
teardownConn(&current_conn);
}
} // namespace sockproxy

23
sockproxy/sockproxy.h Normal file
View File

@@ -0,0 +1,23 @@
#ifndef NS_SOCKPROXY_H
#define NS_SOCKPROXY_H
namespace sockproxy {
/*
* Start a socket proxy between an external TCP connection and the
* child process's stdin/stdout pipes.
*
* Takes FD pointers: on success, the proxy owns the FDs and sets
* them to -1 in the caller. On failure, FDs are untouched and
* the caller remains responsible for closing them.
*
* When both channels finish, the proxy tears itself down and
* calls monitor::stop() to exit the event loop.
*/
typedef void (*on_close_cb_t)(void* data);
bool start(int* connfd, int* pipe_in, int* pipe_out, on_close_cb_t cb, void* data);
void stop();
} // namespace sockproxy
#endif /* NS_SOCKPROXY_H */

527
subproc.cc
View File

@@ -26,7 +26,6 @@
#include <limits.h>
#include <linux/sched.h>
#include <sched.h>
#include <setjmp.h>
#include <signal.h>
#include <stdbool.h>
#include <stddef.h>
@@ -34,6 +33,7 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/prctl.h>
#include <sys/socket.h>
#include <sys/syscall.h>
#include <sys/types.h>
@@ -42,6 +42,7 @@
#include <unistd.h>
#include <string>
#include <thread>
#include <vector>
#include "cgroup.h"
@@ -49,23 +50,17 @@
#include "contain.h"
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "monitor.h"
#include "net.h"
#include "nstun/nstun.h"
#include "sandbox.h"
#include "unotify/unotify.h"
#include "user.h"
#include "util.h"
namespace subproc {
#if !defined(CLONE_NEWCGROUP)
#define CLONE_NEWCGROUP 0x02000000
#endif /* !defined(CLONE_NEWCGROUP) */
#if !defined(CLONE_NEWTIME)
#define CLONE_NEWTIME 0x00000080
#endif /* !defined(CLONE_NEWTIME) */
static const std::string cloneFlagsToStr(uint64_t flags) {
static std::string cloneFlagsToStr(uint64_t flags) {
std::string res;
struct {
@@ -77,9 +72,7 @@ static const std::string cloneFlagsToStr(uint64_t flags) {
NS_VALSTR_STRUCT(CLONE_FS),
NS_VALSTR_STRUCT(CLONE_FILES),
NS_VALSTR_STRUCT(CLONE_SIGHAND),
#if !defined(CLONE_PIDFD)
#define CLONE_PIDFD 0x00001000
#endif
NS_VALSTR_STRUCT(CLONE_PIDFD),
NS_VALSTR_STRUCT(CLONE_PTRACE),
NS_VALSTR_STRUCT(CLONE_VFORK),
@@ -100,13 +93,9 @@ static const std::string cloneFlagsToStr(uint64_t flags) {
NS_VALSTR_STRUCT(CLONE_NEWPID),
NS_VALSTR_STRUCT(CLONE_NEWNET),
NS_VALSTR_STRUCT(CLONE_IO),
#if !defined(CLONE_CLEAR_SIGHAND)
#define CLONE_CLEAR_SIGHAND 0x100000000ULL
#endif /* !defined(CLONE_CLEAR_SIGHAND) */
NS_VALSTR_STRUCT(CLONE_CLEAR_SIGHAND),
#if !defined(CLONE_INTO_CGROUP)
#define CLONE_INTO_CGROUP 0x200000000ULL
#endif /* !defined(CLONE_INTO_CGROUP) */
NS_VALSTR_STRUCT(CLONE_INTO_CGROUP),
};
@@ -139,16 +128,14 @@ static bool resetEnv(void) {
/* Unblock all signals */
sigset_t sset;
sigemptyset(&sset);
if (sigprocmask(SIG_SETMASK, &sset, NULL) == -1) {
if (sigprocmask(SIG_SETMASK, &sset, nullptr) == -1) {
PLOG_W("sigprocmask(SIG_SET, empty)");
return false;
}
return true;
}
static const char kSubprocErrorChar = 'E';
static const std::string concatArgs(const std::vector<const char*>& argv) {
static std::string concatArgs(const std::vector<const char*>& argv) {
std::string ret;
for (const auto& s : argv) {
if (s) {
@@ -161,7 +148,17 @@ static const std::string concatArgs(const std::vector<const char*>& argv) {
return ret;
}
static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, int pipefd) {
static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, int ipc_fd) {
auto connstr = net::connToText(netfd, /* remote= */ true, nullptr);
/*
* Set parent death signal early to prevent the child from hanging
* if the parent dies during the IPC handshake or setup.
*/
if (prctl(PR_SET_PDEATHSIG, SIGKILL, 0, 0, 0) == -1) {
PLOG_W("prctl(PR_SET_PDEATHSIG, SIGKILL) failed");
}
if (nsj->njc.has_oom_score_adj()) {
std::string score = std::to_string(nsj->njc.oom_score_adj());
if (!util::writeBufToFile(
@@ -170,45 +167,40 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
}
}
if (!contain::setupFD(nsj, fd_in, fd_out, fd_err)) {
if (!contain::setupFD(nsj, fd_in, fd_out, fd_err, ipc_fd)) {
return;
}
if (!resetEnv()) {
return;
}
if (pipefd == -1) {
if (ipc_fd == -1) {
if (!user::initNsFromParent(nsj, getpid())) {
LOG_E("Couldn't initialize net user namespace");
LOG_E("Couldn't initialize user namespace");
return;
}
if (nsj->njc.use_cgroupv2()) {
if (!cgroup2::initNsFromParent(nsj, getpid())) {
LOG_E("Couldn't initialize net user namespace");
LOG_E("Couldn't initialize cgroup v2 namespace");
return;
}
} else if (!cgroup::initNsFromParent(nsj, getpid())) {
LOG_E("Couldn't initialize net user namespace");
LOG_E("Couldn't initialize cgroup namespace");
return;
}
} else {
if (pipefd != -1) {
if (!net::initChildPreSync(nsj, pipefd)) {
return;
}
char doneChar;
if (util::readFromFd(pipefd, &doneChar, sizeof(doneChar)) !=
sizeof(doneChar)) {
return;
}
if (doneChar != kSubprocDoneChar) {
return;
}
if (!net::initChildPreSync(nsj, ipc_fd)) {
return;
}
uint32_t doneMsg;
if (!util::recvMsg(ipc_fd, &doneMsg) || doneMsg != monitor::MSG_TAG_READY_H2J) {
return;
}
}
if (!contain::containProc(nsj)) {
return;
}
if (!nsj->njc.keep_env()) {
clearenv();
}
@@ -216,7 +208,6 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
putenv(const_cast<char*>(nsj->njc.envar(i).c_str()));
}
auto connstr = net::connToText(netfd, /* remote= */ true, NULL);
LOG_I("Executing %s for '%s'", QC(nsj->njc.exec_bin().path()), connstr.c_str());
std::vector<const char*> argv;
@@ -228,28 +219,23 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
LOG_D("Exec: %s, Args: [%s]", QC(nsj->njc.exec_bin().path()), concatArgs(argv).c_str());
/* Should be the last one in the sequence */
if (!sandbox::applyPolicy(nsj, pipefd)) {
if (!sandbox::applyPolicy(nsj, ipc_fd)) {
return;
}
if (pipefd != -1 && nsj->njc.seccomp_unotify()) {
char doneChar;
if (util::readFromFd(pipefd, &doneChar, sizeof(doneChar)) != sizeof(doneChar)) {
if (ipc_fd != -1) {
if (!util::sendMsg(ipc_fd, monitor::MSG_TAG_READY_J2H)) {
return;
}
if (doneChar != kSubprocDoneChar) {
uint32_t doneMsg;
if (!util::recvMsg(ipc_fd, &doneMsg) || doneMsg != monitor::MSG_TAG_READY_H2J) {
return;
}
}
if (nsj->njc.exec_bin().exec_fd()) {
#if defined(__NR_execveat)
util::syscall(__NR_execveat, nsj->exec_fd, (uintptr_t)"", (uintptr_t)argv.data(),
(uintptr_t)environ, AT_EMPTY_PATH);
#else /* defined(__NR_execveat) */
LOG_E("Your system doesn't support execveat() syscall");
return;
#endif /* defined(__NR_execveat) */
} else {
execv(nsj->njc.exec_bin().path().c_str(), (char* const*)argv.data());
}
@@ -257,24 +243,24 @@ static void newProc(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, in
PLOG_E("execve(%s) failed", QC(nsj->njc.exec_bin().path()));
}
static void addProc(nsj_t* nsj, pid_t pid, int sock) {
static void addProc(nsj_t* nsj, pid_t pid, int sock, int pidfd) {
pids_t p;
p.start = time(NULL);
p.pid = pid;
p.start = time(nullptr);
p.remote_txt = net::connToText(sock, /* remote= */ true, &p.remote_addr);
p.pasta_pid = -1;
char fname[PATH_MAX];
snprintf(fname, sizeof(fname), "/proc/%d/syscall", (int)pid);
p.pid_syscall_fd = TEMP_FAILURE_RETRY(open(fname, O_RDONLY | O_CLOEXEC));
p.pidfd = pidfd;
if (nsj->pids.find(pid) != nsj->pids.end()) {
LOG_F("pid=%d already exists", pid);
}
nsj->pids.insert(std::make_pair(pid, p));
LOG_D("Added pid=%d with start time %u to the queue for IP: '%s'", pid,
(unsigned int)p.start, p.remote_txt.c_str());
nsj->pids.emplace(pid, std::move(p));
}
static void removeProc(nsj_t* nsj, pid_t pid) {
@@ -283,16 +269,28 @@ static void removeProc(nsj_t* nsj, pid_t pid) {
return;
}
const auto& p = nsj->pids[pid];
auto& p = nsj->pids[pid];
if (p.pasta_pid > 0) {
LOG_D("Killing pasta pid=%d", p.pasta_pid);
kill(p.pasta_pid, SIGKILL);
while (waitpid(p.pasta_pid, nullptr, 0) == -1 && errno == EINTR);
TEMP_FAILURE_RETRY(waitpid(p.pasta_pid, nullptr, 0));
}
LOG_D("Removed pid=%d from the queue (IP:'%s', start time:'%s')", pid, p.remote_txt.c_str(),
util::timeToStr(p.start).c_str());
close(p.pid_syscall_fd);
if (p.thread.joinable()) {
if (std::this_thread::get_id() == p.thread.get_id()) {
p.thread.detach();
} else {
p.thread.join();
}
}
/* pidfd is owned by pids_t - close it here on removal AFTER joining thread */
if (p.pidfd >= 0) {
close(p.pidfd);
}
nsj->pids.erase(pid);
}
@@ -302,214 +300,192 @@ int countProc(nsj_t* nsj) {
void displayProc(nsj_t* nsj) {
LOG_I("Total number of spawned namespaces: %d", countProc(nsj));
time_t now = time(NULL);
time_t now;
now = time(nullptr);
for (const auto& pid : nsj->pids) {
time_t diff = now - pid.second.start;
uint64_t left = nsj->njc.time_limit() ? nsj->njc.time_limit() - (uint64_t)diff : 0;
LOG_I("pid=%d, Remote host: %s, Run time: %ld sec. (time left: %s s.)", pid.first,
pid.second.remote_txt.c_str(), (long)diff,
nsj->njc.time_limit() ? std::to_string(left).c_str() : "unlimited");
}
}
static void seccompViolation(nsj_t* nsj, siginfo_t* si) {
LOG_W("pid=%d committed a syscall/seccomp violation and exited with SIGSYS", si->si_pid);
const auto& p = nsj->pids.find(si->si_pid);
if (p == nsj->pids.end()) {
LOG_W("pid=%d SiStatus:%d SiUid:%d SiUtime:%ld SiStime:%ld (If "
"SiStatus==31 (SIGSYS), then see 'dmesg' or 'journalctl -ek' for possible "
"auditd report with more data)",
(int)si->si_pid, si->si_status, si->si_uid, (long)si->si_utime,
(long)si->si_stime);
LOG_E("Couldn't find pid element in the subproc list for pid=%d", (int)si->si_pid);
return;
}
char buf[4096];
ssize_t rdsize = util::readFromFd(p->second.pid_syscall_fd, buf, sizeof(buf) - 1);
if (rdsize < 1) {
LOG_W("pid=%d SiStatus:%d SiUid:%d SiUtime:%ld SiStime:%ld (If "
"SiStatus==31 (SIGSYS), then see 'dmesg' or 'journalctl -ek' for possible "
"auditd report with more data)",
(int)si->si_pid, si->si_status, si->si_uid, (long)si->si_utime,
(long)si->si_stime);
return;
}
buf[rdsize - 1] = '\0';
uintptr_t arg1, arg2, arg3, arg4, arg5, arg6, sp, pc;
ptrdiff_t sc;
int ret = sscanf(buf, "%td %tx %tx %tx %tx %tx %tx %tx %tx", &sc, &arg1, &arg2, &arg3,
&arg4, &arg5, &arg6, &sp, &pc);
if (ret == 9) {
LOG_W("pid=%d, Syscall number:%td, Arguments:%#tx, %#tx, %#tx, %#tx, %#tx, %#tx, "
"SP:%#tx, PC:%#tx, si_status:%d",
(int)si->si_pid, sc, arg1, arg2, arg3, arg4, arg5, arg6, sp, pc, si->si_status);
} else if (ret == 3) {
LOG_W("pid=%d SiStatus:%d SiUid:%d SiUtime:%ld SiStime:%ld SP:%#tx, PC:%#tx (If "
"SiStatus==31 (SIGSYS), then see 'dmesg' or 'journalctl -ek' for possible "
"auditd report with more data)",
(int)si->si_pid, si->si_status, si->si_uid, (long)si->si_utime,
(long)si->si_stime, arg1, arg2);
return;
} else {
LOG_W("pid=%d SiStatus:%d SiUid:%d SiUtime:%ld SiStime:%ld (If "
"SiStatus==31 (SIGSYS), then see 'dmesg' or 'journalctl -ek' for possible "
"auditd report with more data)",
(int)si->si_pid, si->si_status, si->si_uid, (long)si->si_utime,
(long)si->si_stime);
}
}
static int reapProc(nsj_t* nsj, pid_t pid, bool should_wait = false) {
int status;
if (wait4(pid, &status, should_wait ? 0 : WNOHANG, NULL) == pid) {
if (nsj->njc.use_cgroupv2()) {
cgroup2::finishFromParent(nsj, pid);
if (nsj->njc.time_limit() && (uint64_t)diff < nsj->njc.time_limit()) {
uint64_t left = nsj->njc.time_limit() - (uint64_t)diff;
LOG_I("pid=%d, Remote host: %s, Run time: %ld sec. (time left: %" PRIu64
" s.)",
pid.first, pid.second.remote_txt.c_str(), (long)diff, left);
} else {
cgroup::finishFromParent(nsj, pid);
LOG_I("pid=%d, Remote host: %s, Run time: %ld sec. (time left: %s)",
pid.first, pid.second.remote_txt.c_str(), (long)diff,
nsj->njc.time_limit() ? "expired" : "unlimited");
}
}
}
for (auto& pid_entry : nsj->pids) {
if (pid_entry.second.pasta_pid > 0 && pid == pid_entry.second.pasta_pid) {
int reapProc(nsj_t* nsj, pid_t pid, bool should_wait) {
siginfo_t si;
memset(&si, 0, sizeof(si));
if (TEMP_FAILURE_RETRY(waitid(P_PID, pid, &si, WEXITED | (should_wait ? 0 : WNOHANG))) ==
-1) {
if (errno != ECHILD) {
PLOG_W("waitid(P_PID, id=%d)", pid);
}
return 0;
}
if (si.si_pid == 0) {
return 0;
}
if (nsj->njc.use_cgroupv2()) {
cgroup2::finishFromParent(nsj, pid);
} else {
cgroup::finishFromParent(nsj, pid);
}
for (auto& pid_entry : nsj->pids) {
if (pid_entry.second.pasta_pid > 0 && pid == pid_entry.second.pasta_pid) {
int status = si.si_status;
if (si.si_code == CLD_EXITED) {
LOG_W("Pasta process %d exited unexpectedly with status %d. "
"Killing the jail.",
pid, WEXITSTATUS(status));
kill(pid_entry.first, SIGKILL);
pid_entry.second.pasta_pid = -1;
return WEXITSTATUS(status);
pid, status);
} else {
LOG_W("Pasta process %d terminated by signal %d. Killing the jail.",
pid, status);
status = 128 + status;
}
}
std::string remote_txt = "[UNKNOWN]";
const auto& p = nsj->pids.find(pid);
if (p != nsj->pids.end()) {
remote_txt = p->second.remote_txt;
}
if (WIFEXITED(status)) {
LOG_I("pid=%d (%s) exited with status: %d, (PIDs left: %d)", pid,
remote_txt.c_str(), WEXITSTATUS(status), countProc(nsj) - 1);
removeProc(nsj, pid);
return WEXITSTATUS(status);
}
if (WIFSIGNALED(status)) {
LOG_I("pid=%d (%s) terminated with signal: %s (%d), (PIDs left: %d)", pid,
remote_txt.c_str(), util::sigName(WTERMSIG(status)).c_str(),
WTERMSIG(status), countProc(nsj) - 1);
removeProc(nsj, pid);
return 128 + WTERMSIG(status);
util::syscall(
__NR_pidfd_send_signal, pid_entry.second.pidfd, SIGKILL, 0, 0);
pid_entry.second.pasta_pid = -1;
return status;
}
}
std::string remote_txt = "[UNKNOWN]";
const auto& p = nsj->pids.find(pid);
if (p != nsj->pids.end()) {
remote_txt = p->second.remote_txt;
}
if (si.si_code == CLD_EXITED) {
nsj->exit_status = si.si_status;
LOG_I("pid=%d (%s) exited with status: %d, (PIDs left: %d)", pid,
remote_txt.c_str(), si.si_status, countProc(nsj) - 1);
removeProc(nsj, pid);
return si.si_status;
}
if (si.si_code == CLD_KILLED || si.si_code == CLD_DUMPED) {
nsj->exit_status = 128 + si.si_status;
LOG_I("pid=%d (%s) terminated with signal: %s (%d), (PIDs left: %d)", pid,
remote_txt.c_str(), util::sigName(si.si_status).c_str(), si.si_status,
countProc(nsj) - 1);
removeProc(nsj, pid);
return 128 + si.si_status;
}
LOG_W("pid=%d exited with unexpected si_code=%d, cleaning up", pid, si.si_code);
nsj->exit_status = 1;
removeProc(nsj, pid);
return 0;
}
int reapProc(nsj_t* nsj) {
uint64_t checkTimeouts(
nsj_t* nsj, pid_t target_pid, time_t start_time, const std::string& remote_txt, int pidfd) {
if (nsj->njc.time_limit() == 0) {
return -1; /* no timeout */
}
time_t now = time(nullptr);
time_t diff = now - start_time;
if ((uint64_t)diff >= nsj->njc.time_limit()) {
LOG_I("pid=%d run time >= time limit (%ld >= %" PRIu64 ") (%s). Killing it",
target_pid, (long)diff, (uint64_t)nsj->njc.time_limit(), remote_txt.c_str());
if (pidfd >= 0) {
util::syscall(__NR_pidfd_send_signal, pidfd, SIGCONT, 0, 0);
util::syscall(__NR_pidfd_send_signal, pidfd, SIGKILL, 0, 0);
} else {
kill(target_pid, SIGCONT);
kill(target_pid, SIGKILL);
}
return -1; /* Wait for pidfd to trigger */
}
uint64_t remaining = nsj->njc.time_limit() - diff;
return remaining * 1000; /* in milliseconds */
}
int reapAll(nsj_t* nsj) {
int rv = 0;
siginfo_t si;
for (;;) {
si.si_pid = 0;
if (waitid(P_ALL, 0, &si, WNOHANG | WNOWAIT | WEXITED) == -1) {
if (TEMP_FAILURE_RETRY(waitid(P_ALL, 0, &si, WNOHANG | WNOWAIT | WEXITED)) == -1) {
break;
}
if (si.si_pid == 0) {
break;
}
if (si.si_code == CLD_KILLED && si.si_status == SIGSYS) {
seccompViolation(nsj, &si);
}
rv = reapProc(nsj, si.si_pid);
}
time_t now = time(NULL);
for (const auto& p : nsj->pids) {
if (nsj->njc.time_limit() == 0) {
continue;
}
pid_t pid = p.first;
time_t diff = now - p.second.start;
if ((uint64_t)diff >= nsj->njc.time_limit()) {
LOG_I("pid=%d run time >= time limit (%ld >= %" PRIu64 ") (%s). Killing it",
pid, (long)diff, (uint64_t)nsj->njc.time_limit(),
p.second.remote_txt.c_str());
/*
* Probably a kernel bug - some processes cannot be killed with KILL if
* they're namespaced, and in a stopped state
*/
kill(pid, SIGCONT);
LOG_D("Sent SIGCONT to pid=%d", pid);
kill(pid, SIGKILL);
LOG_D("Sent SIGKILL to pid=%d", pid);
}
rv = reapProc(nsj, si.si_pid);
}
return rv;
}
void killAndReapAll(nsj_t* nsj, int signal) {
while (!nsj->pids.empty()) {
pid_t pid = nsj->pids.begin()->first;
if (nsj->pids.begin()->second.pasta_pid > 0) {
kill(nsj->pids.begin()->second.pasta_pid, SIGKILL);
void killAll(nsj_t* nsj, int signal) {
for (auto& pid_entry : nsj->pids) {
pid_t pid = pid_entry.first;
auto& p = pid_entry.second;
if (p.pasta_pid > 0) {
if (kill(p.pasta_pid, SIGKILL) == -1 && errno != ESRCH) {
PLOG_W("kill(pasta_pid)");
}
}
if (kill(pid, signal) == 0) {
reapProc(nsj, pid, true);
if (p.pidfd >= 0) {
if (util::syscall(__NR_pidfd_send_signal, p.pidfd, signal, 0, 0) == -1 &&
errno != ESRCH) {
PLOG_W("pidfd_send_signal(pidfd=%d, sig=%d)", p.pidfd, signal);
}
} else {
removeProc(nsj, pid);
kill(pid, signal);
}
}
}
static bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
if (!net::initParent(nsj, pid, pipefd)) {
LOG_E("Couldn't initialize net namespace for pid=%d", pid);
static bool initParent(nsj_t* nsj, pid_t pid, int ipc_fd) {
if (!net::initParent(nsj, pid, ipc_fd)) {
LOG_W("Couldn't initialize net namespace for pid=%d", pid);
return false;
}
if (nsj->njc.use_cgroupv2()) {
if (!cgroup2::initNsFromParent(nsj, pid)) {
LOG_E("Couldn't initialize cgroup 2 user namespace for pid=%d", pid);
exit(0xff);
return false;
}
} else if (!cgroup::initNsFromParent(nsj, pid)) {
LOG_E("Couldn't initialize cgroup user namespace for pid=%d", pid);
exit(0xff);
return false;
}
if (!user::initNsFromParent(nsj, pid)) {
LOG_E("Couldn't initialize user namespace for pid=%d", pid);
LOG_W("Couldn't initialize user namespace for pid=%d", pid);
return false;
}
if (!util::writeToFd(pipefd, &kSubprocDoneChar, sizeof(kSubprocDoneChar))) {
LOG_E("Couldn't signal the new process via a socketpair");
return false;
}
if (nsj->njc.seccomp_unotify()) {
int unotif_fd = util::recvFd(pipefd);
if (unotif_fd == -1) {
LOG_E("Failed to receive unotif_fd from child");
return false;
}
LOG_D("Received unotif_fd=%d from child", unotif_fd);
if (!unotify::start(nsj, unotif_fd)) {
LOG_E("Failed to start unotify thread");
close(unotif_fd);
return false;
}
if (!util::writeToFd(pipefd, &kSubprocDoneChar, sizeof(kSubprocDoneChar))) {
LOG_E("Couldn't signal the new process via a socketpair");
if (ipc_fd != -1) {
if (!util::sendMsg(ipc_fd, monitor::MSG_TAG_READY_H2J)) {
LOG_W("Couldn't signal the new process via a socketpair");
return false;
}
}
/* unotify setup and second signaling are handled by the monitor thread */
return true;
}
pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) {
pid_t runChild(
nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, int* pidfd_out, int* ipc_fd_out) {
if (!net::limitConns(nsj, netfd)) {
return 0;
}
@@ -537,18 +513,21 @@ pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) {
int sv[2];
if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == -1) {
PLOG_E("socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC) failed");
PLOG_W("socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC) failed");
return -1;
}
int child_fd = sv[0];
int parent_fd = sv[1];
pid_t pid = cloneProc(flags, SIGCHLD);
pid_t pid = cloneProc(flags, SIGCHLD, pidfd_out);
if (pid == 0) {
close(parent_fd);
newProc(nsj, netfd, fd_in, fd_out, fd_err, child_fd);
util::writeToFd(child_fd, &kSubprocErrorChar, sizeof(kSubprocErrorChar));
LOG_F("Launching child process failed");
util::sendMsg(child_fd, monitor::MSG_TAG_ERROR);
LOG_E("Launching child process failed");
pause();
_exit(0xff);
}
close(child_fd);
if (pid == -1) {
@@ -558,46 +537,33 @@ pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) {
errno = saved_errno;
return pid;
}
addProc(nsj, pid, netfd);
addProc(nsj, pid, netfd, *pidfd_out);
if (!initParent(nsj, pid, parent_fd)) {
close(parent_fd);
LOG_W("initParent failed, killing child pid=%d", pid);
if (*pidfd_out >= 0) {
util::syscall(__NR_pidfd_send_signal, *pidfd_out, SIGKILL, 0, 0);
} else {
kill(pid, SIGKILL);
}
reapProc(nsj, pid, true /* should_wait */);
return -1;
}
char rcvChar;
while (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) {
if (rcvChar == kSubprocErrorChar) {
LOG_W("Received error message from the child process before it has been "
"executed");
close(parent_fd);
return -1;
}
}
close(parent_fd);
*ipc_fd_out = parent_fd;
return pid;
}
/*
* Will be used inside the child process only, so it's safe to have it in BSS.
* Some CPU archs (e.g. aarch64) must have it aligned. Size: 128 KiB (/2)
* Creates a new process via clone3(2) with CLONE_PIDFD.
*
* clone3() is mandatory - we require pidfd for process lifecycle management.
*
* Returns child pid in the parent, 0 in the child, -1 on error.
* On success, *pidfd receives a file descriptor referring to the child.
*/
static uint8_t cloneStack[128 * 1024] __attribute__((aligned(__BIGGEST_ALIGNMENT__)));
/* Cannot be on the stack, as the child's stack pointer will change after clone() */
static __thread jmp_buf env;
[[noreturn]] static int cloneFunc([[maybe_unused]] void* arg) {
longjmp(env, 1);
LOG_F("Execution past longjmp");
}
/*
* Avoid problems with caching of PID/TID in glibc - when using syscall(__NR_clone) glibc doesn't
* update the internal PID/TID caches, what can lead to invalid values being returned by getpid()
* or incorrect PID/TIDs used in raise()/abort() functions
*/
pid_t cloneProc(uint64_t flags, int exit_signal) {
pid_t cloneProc(uint64_t flags, int exit_signal, int* pidfd) {
exit_signal &= CSIGNAL;
if (flags & CLONE_VM) {
@@ -607,50 +573,39 @@ pid_t cloneProc(uint64_t flags, int exit_signal) {
}
if (flags & CLONE_NEWTIME) {
LOG_W("CLONE_NEWTIME reuqested, but it's only supported with the unshare() mode "
LOG_W("CLONE_NEWTIME requested, but it's only supported with the unshare() mode "
"(-Me)");
}
#if defined(__NR_clone3)
struct clone_args ca = {};
ca.exit_signal = (uint64_t)exit_signal;
ca.flags = flags | CLONE_PIDFD | CLONE_CLEAR_SIGHAND;
ca.pidfd = (uint64_t)pidfd;
ca.flags = flags | CLONE_CLEAR_SIGHAND;
pid_t ret = util::syscall(__NR_clone3, (uintptr_t)&ca, sizeof(ca));
if (ret != -1) {
return ret;
}
/*
* Now try without CLONE_CLEAR_SIGHAND as it's supported since Linux 5.5, while clone3
* appeared in Linux 5.3
*/
ca.flags = flags;
ret = util::syscall(__NR_clone3, (uintptr_t)&ca, sizeof(ca));
if (ret != -1 || errno != ENOSYS) {
return ret;
}
#endif /* defined(__NR_clone3) */
if (flags & CLONE_NEWTIME) {
LOG_E("CLONE_NEWTIME was requested but clone3() is not supported");
errno = 0;
if (ret == -1) {
PLOG_W("clone3(flags=%s|CLONE_PIDFD) failed", cloneFlagsToStr(flags).c_str());
return -1;
}
return ret;
}
if (setjmp(env) == 0) {
LOG_D("Cloning process with flags:%s", cloneFlagsToStr(flags).c_str());
/*
* Avoid the problem of the stack growing up/down under different CPU architectures,
* by using middle of the static stack buffer (which is temporary, and used only
* inside of the cloneFunc()
*/
void* stack = &cloneStack[sizeof(cloneStack) / 2];
/* Parent */
return clone(cloneFunc, stack, (int)flags | exit_signal, NULL, NULL, NULL);
/*
* Lightweight clone3 wrapper for internal helpers (mnt, pid) that need
* specific flags (e.g. CLONE_FS) but no pidfd tracking.
*/
pid_t cloneProcNoPidfd(uint64_t flags, int exit_signal) {
exit_signal &= CSIGNAL;
struct clone_args ca = {};
ca.exit_signal = (uint64_t)exit_signal;
ca.flags = flags;
pid_t ret = util::syscall(__NR_clone3, (uintptr_t)&ca, sizeof(ca));
if (ret == -1) {
PLOG_W("clone3(flags=%s) failed", cloneFlagsToStr(flags).c_str());
}
/* Child */
return 0;
return ret;
}
int systemExe(const std::vector<std::string>& args, char** env) {
@@ -668,9 +623,8 @@ int systemExe(const std::vector<std::string>& args, char** env) {
return -1;
}
pid_t pid = fork();
pid_t pid = cloneProcNoPidfd(0, SIGCHLD);
if (pid == -1) {
PLOG_W("fork()");
close(sv[0]);
close(sv[1]);
return -1;
@@ -681,7 +635,7 @@ int systemExe(const std::vector<std::string>& args, char** env) {
execve(argv[0], (char* const*)argv.data(), (char* const*)env);
PLOG_W("execve('%s')", argv[0]);
util::writeToFd(sv[1], "A", 1);
exit(0);
_exit(0);
}
close(sv[1]);
@@ -694,12 +648,9 @@ int systemExe(const std::vector<std::string>& args, char** env) {
for (;;) {
int status;
int ret = wait4(pid, &status, __WALL, NULL);
if (ret == -1 && errno == EINTR) {
continue;
}
int ret = TEMP_FAILURE_RETRY(waitpid(pid, &status, __WALL));
if (ret == -1) {
PLOG_W("wait4(pid=%d)", pid);
PLOG_W("waitpid(pid=%d)", pid);
return -1;
}
if (WIFEXITED(status)) {

17
subproc.h
View File

@@ -29,20 +29,25 @@
#include <string>
#include <vector>
#include "monitor.h"
#include "nsjail.h"
namespace subproc {
constexpr char kSubprocDoneChar = 'D';
/* 0 - network connection limit reached, -1 - error */
pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err);
pid_t runChild(
nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err, int* pidfd_out, int* ipc_fd_out);
int countProc(nsj_t* nsj);
void displayProc(nsj_t* nsj);
void killAndReapAll(nsj_t* nsj, int signal);
void killAll(nsj_t* nsj, int signal);
/* Returns the exit code of the first failing subprocess, or 0 if none fail */
int reapProc(nsj_t* nsj);
int reapAll(nsj_t* nsj);
int reapProc(nsj_t* nsj, pid_t pid, bool should_wait = false);
uint64_t checkTimeouts(
nsj_t* nsj, pid_t target_pid, time_t start_time, const std::string& remote_txt, int pidfd);
int systemExe(const std::vector<std::string>& args, char** env);
pid_t cloneProc(uint64_t flags, int exit_signal);
pid_t cloneProc(uint64_t flags, int exit_signal, int* pidfd);
pid_t cloneProcNoPidfd(uint64_t flags, int exit_signal);
} // namespace subproc

12
unotify/record.h
View File

@@ -3,7 +3,6 @@
#include <string>
#include <tuple>
#include <vector>
#include "unotify/unotify.pb.h"
@@ -40,20 +39,21 @@ struct ResourceRecord {
/* Lexicographical comparison to support std::map usage in stats.cc */
bool operator<(const ResourceRecord& o) const {
return std::tie(has_path1, path1, has_path2, path2, has_net, net_type, net_endpoint, has_net_port, net_port, has_net_path, net_path) <
std::tie(o.has_path1, o.path1, o.has_path2, o.path2, o.has_net, o.net_type, o.net_endpoint, o.has_net_port, o.net_port, o.has_net_path, o.net_path);
return std::tie(has_path1, path1, has_path2, path2, has_net, net_type, net_endpoint,
has_net_port, net_port, has_net_path, net_path) <
std::tie(o.has_path1, o.path1, o.has_path2, o.path2, o.has_net, o.net_type,
o.net_endpoint, o.has_net_port, o.net_port, o.has_net_path, o.net_path);
}
};
struct SyscallRecord {
std::string name;
std::vector<std::string> args;
std::string args_str;
ResourceRecord res;
/* Lexicographical comparison to support std::map usage in stats.cc */
bool operator<(const SyscallRecord& other) const {
return std::tie(name, args, res) <
std::tie(other.name, other.args, other.res);
return std::tie(name, args_str, res) < std::tie(other.name, other.args_str, other.res);
}
};

200
unotify/stats.cc
View File

@@ -4,7 +4,6 @@
#include <google/protobuf/text_format.h>
#include <unistd.h>
#include <map>
#include <mutex>
#include "logs.h"
@@ -13,13 +12,31 @@
namespace unotify {
static std::mutex stats_mu;
/* Aggregates syscall statistics on the fly.
* Uses std::map which requires SyscallRecord to have operator<. */
static std::map<SyscallRecord, size_t> stats;
struct StatEntry {
SyscallRecord rec;
size_t count;
};
constexpr size_t MAX_STATS = 1024;
static StatEntry stats_array[MAX_STATS];
static size_t stats_count = 0;
void addStat(const SyscallRecord& rec) {
std::lock_guard<std::mutex> lock(stats_mu);
stats[rec]++;
for (size_t i = 0; i < stats_count; ++i) {
if (!(stats_array[i].rec < rec) && !(rec < stats_array[i].rec)) {
stats_array[i].count++;
return;
}
}
if (stats_count < MAX_STATS) {
stats_array[stats_count].rec = rec;
stats_array[stats_count].count = 1;
stats_count++;
} else {
LOG_W("Max stats reached, dropping record");
}
}
static void fillPathInfoPb(Stat_Path* pb, const PathInfoRecord& rec) {
@@ -61,56 +78,150 @@ struct NetInfoRecord {
struct SyscallKey {
std::string name;
std::vector<std::string> args;
std::string args_str;
bool operator<(const SyscallKey& o) const {
if (name != o.name) return name < o.name;
return args < o.args;
return args_str < o.args_str;
}
};
constexpr size_t MAX_SYSCALLS_PER_RESOURCE = 64;
struct SyscallKeyEntry {
SyscallKey key;
uint64_t count;
};
struct FsStats {
uint64_t count = 0;
std::map<SyscallKey, uint64_t> syscalls;
SyscallKeyEntry syscalls[MAX_SYSCALLS_PER_RESOURCE];
size_t syscalls_count = 0;
};
struct NetStats {
uint64_t count = 0;
std::map<SyscallKey, uint64_t> syscalls;
SyscallKeyEntry syscalls[MAX_SYSCALLS_PER_RESOURCE];
size_t syscalls_count = 0;
};
struct FsStatsEntry {
PathInfoRecord path_rec;
FsStats stats;
};
struct NetStatsEntry {
NetInfoRecord net_rec;
NetStats stats;
};
constexpr size_t MAX_FS_STATS = 256;
constexpr size_t MAX_NET_STATS = 256;
static FsStats* find_or_insert_fs(FsStatsEntry* array, size_t* count, const PathInfoRecord& key) {
for (size_t i = 0; i < *count; ++i) {
if (!(array[i].path_rec < key) && !(key < array[i].path_rec)) {
return &array[i].stats;
}
}
if (*count < MAX_FS_STATS) {
array[*count].path_rec = key;
FsStats* stats = &array[*count].stats;
stats->count = 0;
stats->syscalls_count = 0;
(*count)++;
return stats;
}
return nullptr;
}
static NetStats* find_or_insert_net(NetStatsEntry* array, size_t* count, const NetInfoRecord& key) {
for (size_t i = 0; i < *count; ++i) {
if (!(array[i].net_rec < key) && !(key < array[i].net_rec)) {
return &array[i].stats;
}
}
if (*count < MAX_NET_STATS) {
array[*count].net_rec = key;
NetStats* stats = &array[*count].stats;
stats->count = 0;
stats->syscalls_count = 0;
(*count)++;
return stats;
}
return nullptr;
}
static SyscallKeyEntry* find_or_insert_syscall(
SyscallKeyEntry* array, size_t* count, const SyscallKey& key) {
for (size_t i = 0; i < *count; ++i) {
if (!(array[i].key < key) && !(key < array[i].key)) {
return &array[i];
}
}
if (*count < MAX_SYSCALLS_PER_RESOURCE) {
array[*count].key = key;
array[*count].count = 0;
SyscallKeyEntry* entry = &array[*count];
(*count)++;
return entry;
}
return nullptr;
}
void printStats(nsj_t* nsj) {
if (!nsj->njc.seccomp_unotify()) {
return;
}
if (stats.empty()) {
return; // Do not emit if empty
}
std::map<PathInfoRecord, FsStats> fs_stats;
std::map<NetInfoRecord, NetStats> net_stats;
static thread_local FsStatsEntry fs_stats_array[MAX_FS_STATS];
size_t fs_stats_count = 0;
static thread_local NetStatsEntry net_stats_array[MAX_NET_STATS];
size_t net_stats_count = 0;
{
std::lock_guard<std::mutex> lock(stats_mu);
for (const auto& [rec, count] : stats) {
SyscallKey sys_key{rec.name, rec.args};
if (stats_count == 0) {
return; // Do not emit if empty
}
for (size_t i = 0; i < stats_count; ++i) {
const auto& rec = stats_array[i].rec;
size_t count = stats_array[i].count;
SyscallKey sys_key{rec.name, rec.args_str};
if (rec.res.has_path1) {
SyscallKey p1_key = sys_key;
if (!rec.res.path1.mode_extra.empty()) {
p1_key.args.push_back(
"mode_extra=" + rec.res.path1.mode_extra);
p1_key.args_str +=
"mode_extra=" + rec.res.path1.mode_extra + " ";
}
FsStats* fs_stat = find_or_insert_fs(
fs_stats_array, &fs_stats_count, rec.res.path1);
if (fs_stat) {
fs_stat->count += count;
SyscallKeyEntry* sys_entry = find_or_insert_syscall(
fs_stat->syscalls, &fs_stat->syscalls_count, p1_key);
if (sys_entry) {
sys_entry->count += count;
}
}
fs_stats[rec.res.path1].count += count;
fs_stats[rec.res.path1].syscalls[p1_key] += count;
}
if (rec.res.has_path2) {
SyscallKey p2_key = sys_key;
if (!rec.res.path2.mode_extra.empty()) {
p2_key.args.push_back(
"mode_extra=" + rec.res.path2.mode_extra);
p2_key.args_str +=
"mode_extra=" + rec.res.path2.mode_extra + " ";
}
FsStats* fs_stat = find_or_insert_fs(
fs_stats_array, &fs_stats_count, rec.res.path2);
if (fs_stat) {
fs_stat->count += count;
SyscallKeyEntry* sys_entry = find_or_insert_syscall(
fs_stat->syscalls, &fs_stat->syscalls_count, p2_key);
if (sys_entry) {
sys_entry->count += count;
}
}
fs_stats[rec.res.path2].count += count;
fs_stats[rec.res.path2].syscalls[p2_key] += count;
}
if (rec.res.has_net) {
NetInfoRecord net_rec;
@@ -121,33 +232,42 @@ void printStats(nsj_t* nsj) {
net_rec.has_path = rec.res.has_net_path;
net_rec.path = rec.res.net_path;
net_stats[net_rec].count += count;
net_stats[net_rec].syscalls[sys_key] += count;
NetStats* net_stat =
find_or_insert_net(net_stats_array, &net_stats_count, net_rec);
if (net_stat) {
net_stat->count += count;
SyscallKeyEntry* sys_entry = find_or_insert_syscall(
net_stat->syscalls, &net_stat->syscalls_count, sys_key);
if (sys_entry) {
sys_entry->count += count;
}
}
}
}
}
Stat report_pb;
for (const auto& [path_rec, fs_stat] : fs_stats) {
for (size_t i = 0; i < fs_stats_count; ++i) {
const auto& path_rec = fs_stats_array[i].path_rec;
const auto& fs_stat = fs_stats_array[i].stats;
Stat_Path* fs_pb = report_pb.add_fs_access();
fs_pb->set_count(fs_stat.count);
fillPathInfoPb(fs_pb, path_rec);
for (const auto& [sys_key, sys_count] : fs_stat.syscalls) {
for (size_t j = 0; j < fs_stat.syscalls_count; ++j) {
Stat_Syscall* sys_pb = fs_pb->add_syscall();
sys_pb->set_name(sys_key.name);
sys_pb->set_count(sys_count);
for (const auto& arg : sys_key.args) {
sys_pb->add_args(arg);
}
sys_pb->set_name(fs_stat.syscalls[j].key.name);
sys_pb->set_count(fs_stat.syscalls[j].count);
sys_pb->add_args(fs_stat.syscalls[j].key.args_str);
}
}
for (const auto& [net_rec, net_stat] : net_stats) {
for (size_t i = 0; i < net_stats_count; ++i) {
const auto& net_rec = net_stats_array[i].net_rec;
const auto& net_stat = net_stats_array[i].stats;
Stat_NetResource* net_pb = report_pb.add_net_access();
net_pb->set_count(net_stat.count);
net_pb->set_type(net_rec.type);
if (!net_rec.endpoint.empty()) {
net_pb->set_endpoint(net_rec.endpoint);
@@ -159,13 +279,11 @@ void printStats(nsj_t* nsj) {
fillPathInfoPb(net_pb->mutable_socket_path(), net_rec.path);
}
for (const auto& [sys_key, sys_count] : net_stat.syscalls) {
for (size_t j = 0; j < net_stat.syscalls_count; ++j) {
Stat_Syscall* sys_pb = net_pb->add_syscall();
sys_pb->set_name(sys_key.name);
sys_pb->set_count(sys_count);
for (const auto& arg : sys_key.args) {
sys_pb->add_args(arg);
}
sys_pb->set_name(net_stat.syscalls[j].key.name);
sys_pb->set_count(net_stat.syscalls[j].count);
sys_pb->add_args(net_stat.syscalls[j].key.args_str);
}
}

280
unotify/syscall.cc
View File

@@ -15,6 +15,7 @@
#include "logs.h"
#include "macros.h"
#include "missing_defs.h"
#include "unotify/syscall_defs.h"
#include "util.h"
@@ -26,15 +27,18 @@ constexpr size_t kMaxPathLen = 4096;
/* Helper functions string/memory reading, path resolution, etc. */
static std::string getSocketType(pid_t pid, int target_fd) {
int pidfd = syscall(__NR_pidfd_open, pid, 0);
int pidfd = util::syscall(__NR_pidfd_open, pid, 0);
if (pidfd < 0) {
return "";
}
int local_fd = syscall(__NR_pidfd_getfd, pidfd, target_fd, 0);
fcntl(pidfd, F_SETFD, FD_CLOEXEC);
int local_fd = util::syscall(__NR_pidfd_getfd, pidfd, target_fd, 0);
close(pidfd);
if (local_fd < 0) {
return "";
}
fcntl(local_fd, F_SETFD, FD_CLOEXEC);
int type = 0, proto = 0;
socklen_t len = sizeof(type);
@@ -57,7 +61,9 @@ static std::string getSocketType(pid_t pid, int target_fd) {
break;
}
}
if (!found) type_str = "SOCK_TYPE_" + std::to_string(type);
if (!found) {
type_str = "SOCK_TYPE_" + std::to_string(type);
}
}
len = sizeof(proto);
if (getsockopt(local_fd, SOL_SOCKET, SO_PROTOCOL, &proto, &len) == 0) {
@@ -80,62 +86,88 @@ static std::string getSocketType(pid_t pid, int target_fd) {
break;
}
}
if (!found && proto != 0) type_str += " (proto_" + std::to_string(proto) + ")";
if (!found && proto != 0) {
type_str += " (proto_" + std::to_string(proto) + ")";
}
}
close(local_fd);
return type_str;
}
static std::string readStringFromMem(pid_t pid, uint64_t addr) {
if (addr == 0) return "NULL";
char buf[kMaxPathLen];
struct iovec local = {buf, sizeof(buf) - 1};
struct iovec remote = {(void*)addr, sizeof(buf) - 1};
if (addr == 0) {
return "NULL";
}
/* Heap-allocate to avoid stack pressure, but avoid static buffer reuse
* in case it causes subtle issues with event interleaving */
std::string buf(kMaxPathLen - 1, '\0');
struct iovec local = {.iov_base = buf.data(), .iov_len = buf.size()};
struct iovec remote = {.iov_base = (void*)addr, .iov_len = buf.size()};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) {
return "<invalid_ptr>";
}
buf[ret] = '\0';
return std::string(buf);
buf.resize(strnlen(buf.data(), ret));
return buf;
}
static std::vector<std::string> readStringArrayFromMem(pid_t pid, uint64_t addr, bool is_32bit) {
std::vector<std::string> arr;
if (addr == 0) return arr;
static void appendStringArrayFromMem(
pid_t pid, uint64_t addr, bool is_32bit, const char* prefix, std::string& out) {
if (addr == 0) {
return;
}
/* Heap-allocate pointer arrays to avoid stack pressure */
if (is_32bit) {
uint32_t ptrs[kMaxArgs];
struct iovec local = {ptrs, sizeof(ptrs)};
struct iovec remote = {(void*)addr, sizeof(ptrs)};
std::vector<uint32_t> ptrs(kMaxArgs);
struct iovec local = {
.iov_base = ptrs.data(), .iov_len = ptrs.size() * sizeof(uint32_t)};
struct iovec remote = {
.iov_base = (void*)addr, .iov_len = ptrs.size() * sizeof(uint32_t)};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) return arr;
if (ret <= 0) {
return;
}
int num_ptrs = ret / sizeof(uint32_t);
for (int i = 0; i < num_ptrs; i++) {
if (ptrs[i] == 0) break;
arr.push_back(readStringFromMem(pid, ptrs[i]));
if (ptrs[i] == 0) {
break;
}
out += prefix;
out +=
"[" + std::to_string(i) + "]=" + readStringFromMem(pid, ptrs[i]) + " ";
}
} else {
uint64_t ptrs[kMaxArgs];
struct iovec local = {ptrs, sizeof(ptrs)};
struct iovec remote = {(void*)addr, sizeof(ptrs)};
std::vector<uint64_t> ptrs(kMaxArgs);
struct iovec local = {
.iov_base = ptrs.data(), .iov_len = ptrs.size() * sizeof(uint64_t)};
struct iovec remote = {
.iov_base = (void*)addr, .iov_len = ptrs.size() * sizeof(uint64_t)};
ssize_t ret = process_vm_readv(pid, &local, 1, &remote, 1, 0);
if (ret <= 0) return arr;
if (ret <= 0) {
return;
}
int num_ptrs = ret / sizeof(uint64_t);
for (int i = 0; i < num_ptrs; i++) {
if (ptrs[i] == 0) break;
arr.push_back(readStringFromMem(pid, ptrs[i]));
if (ptrs[i] == 0) {
break;
}
out += prefix;
out +=
"[" + std::to_string(i) + "]=" + readStringFromMem(pid, ptrs[i]) + " ";
}
}
return arr;
}
static std::string getAbsPath(pid_t pid, int dirfd, const std::string& raw_path) {
if (raw_path.empty() || raw_path[0] == '/') return raw_path;
if (raw_path.empty() || raw_path[0] == '/') {
return raw_path;
}
std::string link_path;
if (dirfd == AT_FDCWD || dirfd == -100) {
@@ -144,26 +176,31 @@ static std::string getAbsPath(pid_t pid, int dirfd, const std::string& raw_path)
link_path = "/proc/" + std::to_string(pid) + "/fd/" + std::to_string(dirfd);
}
char buf[PATH_MAX];
ssize_t len = readlink(link_path.c_str(), buf, sizeof(buf) - 1);
if (len <= 0) return raw_path; // fallback
buf[len] = '\0';
/* Heap-allocate to avoid stack pressure */
std::string buf(PATH_MAX - 1, '\0');
ssize_t len = readlink(link_path.c_str(), buf.data(), buf.size());
if (len <= 0) {
return raw_path; // fallback
}
std::string abs_path = buf;
if (abs_path.back() != '/') abs_path += '/';
std::string abs_path(buf.data(), len);
if (abs_path.back() != '/') {
abs_path += '/';
}
abs_path += raw_path;
return abs_path;
}
static void getFileMode(int flags, PathInfoRecord* out) {
if ((flags & O_ACCMODE) == O_RDONLY)
if ((flags & O_ACCMODE) == O_RDONLY) {
out->mode = Stat_Path_Mode_RDONLY;
else if ((flags & O_ACCMODE) == O_WRONLY)
} else if ((flags & O_ACCMODE) == O_WRONLY) {
out->mode = Stat_Path_Mode_WRONLY;
else if ((flags & O_ACCMODE) == O_RDWR)
} else if ((flags & O_ACCMODE) == O_RDWR) {
out->mode = Stat_Path_Mode_RDWR;
else
} else {
out->mode = Stat_Path_Mode_UNSPECIFIED;
}
std::string mode_extra;
struct {
@@ -200,12 +237,22 @@ static void getFileMode(int flags, PathInfoRecord* out) {
}
static std::string getAccessMode(int mode) {
if (mode == F_OK) return "F_OK";
if (mode == F_OK) {
return "F_OK";
}
std::string acc;
if (mode & R_OK) acc += "R_OK|";
if (mode & W_OK) acc += "W_OK|";
if (mode & X_OK) acc += "X_OK|";
if (!acc.empty()) acc.pop_back();
if (mode & R_OK) {
acc += "R_OK|";
}
if (mode & W_OK) {
acc += "W_OK|";
}
if (mode & X_OK) {
acc += "X_OK|";
}
if (!acc.empty()) {
acc.pop_back();
}
return acc;
}
@@ -214,13 +261,27 @@ static Stat_Path_Type getStatInfo(const std::string& path) {
if (lstat(path.c_str(), &st) == -1) {
return Stat_Path_Type_NONEXISTENT;
}
if (S_ISREG(st.st_mode)) return Stat_Path_Type_REGULAR;
if (S_ISDIR(st.st_mode)) return Stat_Path_Type_DIR;
if (S_ISCHR(st.st_mode)) return Stat_Path_Type_CHR;
if (S_ISBLK(st.st_mode)) return Stat_Path_Type_BLK;
if (S_ISFIFO(st.st_mode)) return Stat_Path_Type_FIFO;
if (S_ISLNK(st.st_mode)) return Stat_Path_Type_LINK;
if (S_ISSOCK(st.st_mode)) return Stat_Path_Type_SOCK;
if (S_ISREG(st.st_mode)) {
return Stat_Path_Type_REGULAR;
}
if (S_ISDIR(st.st_mode)) {
return Stat_Path_Type_DIR;
}
if (S_ISCHR(st.st_mode)) {
return Stat_Path_Type_CHR;
}
if (S_ISBLK(st.st_mode)) {
return Stat_Path_Type_BLK;
}
if (S_ISFIFO(st.st_mode)) {
return Stat_Path_Type_FIFO;
}
if (S_ISLNK(st.st_mode)) {
return Stat_Path_Type_LINK;
}
if (S_ISSOCK(st.st_mode)) {
return Stat_Path_Type_SOCK;
}
return Stat_Path_Type_UNKNOWN;
}
@@ -235,11 +296,13 @@ static void populatePathInfo(pid_t pid, int dirfd, uint64_t addr, PathInfoRecord
static void parseSockaddr(struct seccomp_notif* req, SyscallRecord* rec, uint64_t addr,
socklen_t addrlen, const std::string& socket_type_str) {
if (addrlen > sizeof(struct sockaddr_storage) || addr == 0) return;
if (addrlen > sizeof(struct sockaddr_storage) || addr == 0) {
return;
}
struct sockaddr_storage ss = {};
struct iovec local = {&ss, addrlen};
struct iovec remote = {(void*)addr, addrlen};
struct iovec local = {.iov_base = &ss, .iov_len = addrlen};
struct iovec remote = {.iov_base = (void*)addr, .iov_len = addrlen};
ssize_t read_bytes = process_vm_readv(req->pid, &local, 1, &remote, 1, 0);
if (read_bytes >= (ssize_t)sizeof(sa_family_t)) {
char host[INET6_ADDRSTRLEN] = "unknown";
@@ -334,7 +397,9 @@ static std::string getDomainStr(int domain) {
NS_VALSTR_STRUCT(AF_VSOCK),
};
for (const auto& i : domains) {
if (domain == i.val) return i.name;
if (domain == i.val) {
return i.name;
}
}
return std::to_string(domain);
}
@@ -362,14 +427,22 @@ static std::string getTypeStr(int type) {
break;
}
}
if (!found) res = std::to_string(base_type);
if (type & SOCK_CLOEXEC) res += "|SOCK_CLOEXEC";
if (type & SOCK_NONBLOCK) res += "|SOCK_NONBLOCK";
if (!found) {
res = std::to_string(base_type);
}
if (type & SOCK_CLOEXEC) {
res += "|SOCK_CLOEXEC";
}
if (type & SOCK_NONBLOCK) {
res += "|SOCK_NONBLOCK";
}
return res;
}
static std::string getProtocolStr(int proto) {
if (proto == 0) return "0";
if (proto == 0) {
return "0";
}
struct {
const int val;
const char* const name;
@@ -402,7 +475,9 @@ static std::string getProtocolStr(int proto) {
NS_VALSTR_STRUCT(IPPROTO_ICMPV6),
};
for (const auto& i : protos) {
if (proto == i.val) return i.name;
if (proto == i.val) {
return i.name;
}
}
return std::to_string(proto);
}
@@ -418,10 +493,14 @@ static void decodeSyscallArgs(
bool is_32bit = false;
#ifdef AUDIT_ARCH_I386
if (req->data.arch == AUDIT_ARCH_I386) is_32bit = true;
if (req->data.arch == AUDIT_ARCH_I386) {
is_32bit = true;
}
#endif
#ifdef AUDIT_ARCH_ARM
if (req->data.arch == AUDIT_ARCH_ARM) is_32bit = true;
if (req->data.arch == AUDIT_ARCH_ARM) {
is_32bit = true;
}
#endif
for (int i = 0; i < 6; i++) {
@@ -447,9 +526,10 @@ static void decodeSyscallArgs(
case ArgRole::DIRFD: {
int dirfd = (int)arg;
rec->args.push_back("dirfd=" + (dirfd == AT_FDCWD || dirfd == -100
? std::string("AT_FDCWD")
: std::to_string(dirfd)));
std::string arg_str =
"dirfd=" + (dirfd == AT_FDCWD || dirfd == -100 ? std::string("AT_FDCWD")
: std::to_string(dirfd));
rec->args_str += arg_str + " ";
current_dirfd = dirfd;
break;
}
@@ -463,49 +543,54 @@ static void decodeSyscallArgs(
case ArgRole::OCTAL: {
char buf[32];
snprintf(buf, sizeof(buf), "0%o", (unsigned int)arg);
rec->args.push_back("mode=" + std::string(buf));
std::string arg_str = "mode=" + std::string(buf);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::ACCESS:
rec->args.push_back("mode=" + getAccessMode((int)arg));
case ArgRole::ACCESS: {
std::string arg_str = "mode=" + getAccessMode((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::UID:
rec->args.push_back("owner=" + std::to_string((int)arg));
case ArgRole::UID: {
std::string arg_str = "owner=" + std::to_string((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::GID:
rec->args.push_back("group=" + std::to_string((int)arg));
case ArgRole::GID: {
std::string arg_str = "group=" + std::to_string((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::ARGV: {
auto argv = readStringArrayFromMem(req->pid, arg, is_32bit);
for (size_t j = 0; j < argv.size(); j++) {
rec->args.push_back("argv[" + std::to_string(j) + "]=" + argv[j]);
}
appendStringArrayFromMem(req->pid, arg, is_32bit, "argv", rec->args_str);
break;
}
case ArgRole::ENVP: {
auto envp = readStringArrayFromMem(req->pid, arg, is_32bit);
for (size_t j = 0; j < envp.size(); j++) {
rec->args.push_back("envp[" + std::to_string(j) + "]=" + envp[j]);
}
appendStringArrayFromMem(req->pid, arg, is_32bit, "envp", rec->args_str);
break;
}
case ArgRole::FD: {
rec->args.push_back("fd=" + std::to_string((int)arg));
std::string arg_str = "fd=" + std::to_string((int)arg);
rec->args_str += arg_str + " ";
last_socket_type = getSocketType(req->pid, (int)arg);
if (!last_socket_type.empty()) {
rec->args.push_back("type=" + last_socket_type);
std::string type_str = "type=" + last_socket_type;
rec->args_str += type_str + " ";
}
break;
}
case ArgRole::SADDR: {
if (arg == 0) break;
if (arg == 0) {
break;
}
socklen_t addrlen = (i + 1 < 6) ? (socklen_t)args[i + 1] : 0;
parseSockaddr(req, rec, arg, addrlen, last_socket_type);
break;
@@ -514,21 +599,29 @@ static void decodeSyscallArgs(
case ArgRole::ALEN:
break; /* consumed by preceding SADDR */
case ArgRole::IFLAGS:
rec->args.push_back("flags=" + std::to_string((int)arg));
case ArgRole::IFLAGS: {
std::string arg_str = "flags=" + std::to_string((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::DOMAIN:
rec->args.push_back("domain=" + getDomainStr((int)arg));
case ArgRole::DOMAIN: {
std::string arg_str = "domain=" + getDomainStr((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::STYPE:
rec->args.push_back("type=" + getTypeStr((int)arg));
case ArgRole::STYPE: {
std::string arg_str = "type=" + getTypeStr((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::PROTO:
rec->args.push_back("protocol=" + getProtocolStr((int)arg));
case ArgRole::PROTO: {
std::string arg_str = "protocol=" + getProtocolStr((int)arg);
rec->args_str += arg_str + " ";
break;
}
case ArgRole::OHOW: {
struct {
@@ -536,16 +629,17 @@ static void decodeSyscallArgs(
__u64 mode;
__u64 resolve;
} how = {};
struct iovec local = {&how, sizeof(how)};
struct iovec remote = {(void*)arg, sizeof(how)};
struct iovec local = {.iov_base = &how, .iov_len = sizeof(how)};
struct iovec remote = {.iov_base = (void*)arg, .iov_len = sizeof(how)};
if (process_vm_readv(req->pid, &local, 1, &remote, 1, 0) >=
(ssize_t)sizeof(how.flags)) {
if (last_path) {
getFileMode((int)how.flags, last_path);
}
if (how.resolve != 0) {
rec->args.push_back(
"resolve=" + std::to_string(how.resolve));
std::string arg_str =
"resolve=" + std::to_string(how.resolve);
rec->args_str += arg_str + " ";
}
}
break;

213
unotify/syscall_defs.h
View File

@@ -1,15 +1,16 @@
#ifndef NSJAIL_UNOTIFY_SYSCALL_DEFS_H
#define NSJAIL_UNOTIFY_SYSCALL_DEFS_H
#include <sys/syscall.h>
#include <cstddef>
#include <cstdint>
#include <string>
#include <sys/syscall.h>
#include "../missing_defs.h"
namespace unotify {
enum class SyscallCategory : uint8_t {
FS,
NET,
@@ -24,33 +25,33 @@ enum class SyscallCategory : uint8_t {
* last_path - set by PATH, consumed by FLAGS/OHOW
*/
enum class ArgRole : uint8_t {
SKIP, /* Don't decode this argument */
PATH, /* String ptr -> populatePathInfo (alternates path1/path2) */
DIRFD, /* Directory fd; sets context for next PATH */
FLAGS, /* Open flags -> getFileMode on last populated path */
OCTAL, /* Permission mode -> "mode=0NNN" */
ACCESS, /* Access mode -> "mode=R_OK|W_OK|X_OK" */
UID, /* User id -> "owner=N" */
GID, /* Group id -> "group=N" */
ARGV, /* String array ptr -> "argv[i]=..." */
ENVP, /* String array ptr -> "envp[i]=..." */
FD, /* Socket fd -> "fd=N" + getSocketType */
SADDR, /* sockaddr ptr; next arg must be ALEN */
ALEN, /* sockaddr length; consumed by preceding SADDR */
IFLAGS, /* Integer flags -> "flags=N" */
SKIP, /* Don't decode this argument */
PATH, /* String ptr -> populatePathInfo (alternates path1/path2) */
DIRFD, /* Directory fd; sets context for next PATH */
FLAGS, /* Open flags -> getFileMode on last populated path */
OCTAL, /* Permission mode -> "mode=0NNN" */
ACCESS, /* Access mode -> "mode=R_OK|W_OK|X_OK" */
UID, /* User id -> "owner=N" */
GID, /* Group id -> "group=N" */
ARGV, /* String array ptr -> "argv[i]=..." */
ENVP, /* String array ptr -> "envp[i]=..." */
FD, /* Socket fd -> "fd=N" + getSocketType */
SADDR, /* sockaddr ptr; next arg must be ALEN */
ALEN, /* sockaddr length; consumed by preceding SADDR */
IFLAGS, /* Integer flags -> "flags=N" */
DOMAIN, /* Socket domain -> getDomainStr */
STYPE, /* Socket type -> getTypeStr */
PROTO, /* Protocol -> getProtocolStr */
OHOW, /* struct open_how ptr -> decode flags + resolve */
DOMAIN, /* Socket domain -> getDomainStr */
STYPE, /* Socket type -> getTypeStr */
PROTO, /* Protocol -> getProtocolStr */
OHOW, /* struct open_how ptr -> decode flags + resolve */
};
struct SyscallDef {
int nr; /* __NR_xxx */
const char* kafel_name; /* name for kafel policy ("newstat", "sendmsg") */
const char* display_name; /* name for stats output ("stat", "sendmsg") */
int nr; /* __NR_xxx */
const char* kafel_name; /* name for kafel policy ("newstat", "sendmsg") */
const char* display_name; /* name for stats output ("stat", "sendmsg") */
SyscallCategory category;
ArgRole args[6]; /* role of each syscall argument */
ArgRole args[6]; /* role of each syscall argument */
};
/* Shorthand for table readability */
@@ -63,166 +64,166 @@ using A = ArgRole;
* The kafel BPF policy, name lookup, and argument decoding all derive from this table.
*/
static constexpr SyscallDef kTracedSyscalls[] = {
/* FS - arg0 = path (AT_FDCWD implied) */
/* FS - arg0 = path (AT_FDCWD implied) */
#ifdef __NR_open
{__NR_open, "open", "open", SyscallCategory::FS,
{A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_open, "open", "open", SyscallCategory::FS,
{A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_open */
#ifdef __NR_stat
{__NR_stat, "newstat", "stat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_stat, "newstat", "stat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_stat */
#ifdef __NR_lstat
{__NR_lstat, "newlstat", "lstat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_lstat, "newlstat", "lstat", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_lstat */
#ifdef __NR_access
{__NR_access, "access", "access", SyscallCategory::FS,
{A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_access, "access", "access", SyscallCategory::FS,
{A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_access */
#ifdef __NR_chmod
{__NR_chmod, "chmod", "chmod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chmod, "chmod", "chmod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_chmod */
#ifdef __NR_chown
{__NR_chown, "chown", "chown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chown, "chown", "chown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_chown */
#ifdef __NR_lchown
{__NR_lchown, "lchown", "lchown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
{__NR_lchown, "lchown", "lchown", SyscallCategory::FS,
{A::PATH, A::UID, A::GID, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_lchown */
#ifdef __NR_readlink
{__NR_readlink, "readlink", "readlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_readlink, "readlink", "readlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_readlink */
#ifdef __NR_mkdir
{__NR_mkdir, "mkdir", "mkdir", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mkdir, "mkdir", "mkdir", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_mkdir */
#ifdef __NR_mknod
{__NR_mknod, "mknod", "mknod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mknod, "mknod", "mknod", SyscallCategory::FS,
{A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_mknod */
#ifdef __NR_rmdir
{__NR_rmdir, "rmdir", "rmdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_rmdir, "rmdir", "rmdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_rmdir */
#ifdef __NR_unlink
{__NR_unlink, "unlink", "unlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_unlink, "unlink", "unlink", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_unlink */
#ifdef __NR_execve
{__NR_execve, "execve", "execve", SyscallCategory::FS,
{A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_execve, "execve", "execve", SyscallCategory::FS,
{A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_execve */
#ifdef __NR_chdir
{__NR_chdir, "chdir", "chdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chdir, "chdir", "chdir", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_chdir */
#ifdef __NR_chroot
{__NR_chroot, "chroot", "chroot", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_chroot, "chroot", "chroot", SyscallCategory::FS,
{A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_chroot */
/* FS - arg0 = dirfd, arg1 = path */
/* FS - arg0 = dirfd, arg1 = path */
#ifdef __NR_openat
{__NR_openat, "openat", "openat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP}},
{__NR_openat, "openat", "openat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::FLAGS, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_openat */
#ifdef __NR_openat2
{__NR_openat2, "openat2", "openat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OHOW, A::SKIP, A::SKIP, A::SKIP}},
{__NR_openat2, "openat2", "openat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OHOW, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_openat2 */
#ifdef __NR_newfstatat
{__NR_newfstatat, "newfstatat", "newfstatat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_newfstatat, "newfstatat", "newfstatat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_newfstatat */
#ifdef __NR_faccessat
{__NR_faccessat, "faccessat", "faccessat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP}},
{__NR_faccessat, "faccessat", "faccessat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ACCESS, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_faccessat */
#ifdef __NR_fchmodat
{__NR_fchmodat, "fchmodat", "fchmodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_fchmodat, "fchmodat", "fchmodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_fchmodat */
#ifdef __NR_fchownat
{__NR_fchownat, "fchownat", "fchownat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::UID, A::GID, A::SKIP, A::SKIP}},
{__NR_fchownat, "fchownat", "fchownat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::UID, A::GID, A::SKIP, A::SKIP}},
#endif /* __NR_fchownat */
#ifdef __NR_readlinkat
{__NR_readlinkat, "readlinkat", "readlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_readlinkat, "readlinkat", "readlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_readlinkat */
#ifdef __NR_mkdirat
{__NR_mkdirat, "mkdirat", "mkdirat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mkdirat, "mkdirat", "mkdirat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_mkdirat */
#ifdef __NR_mknodat
{__NR_mknodat, "mknodat", "mknodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
{__NR_mknodat, "mknodat", "mknodat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::OCTAL, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_mknodat */
#ifdef __NR_unlinkat
{__NR_unlinkat, "unlinkat", "unlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_unlinkat, "unlinkat", "unlinkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_unlinkat */
#ifdef __NR_execveat
{__NR_execveat, "execveat", "execveat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP}},
{__NR_execveat, "execveat", "execveat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::ARGV, A::ENVP, A::SKIP, A::SKIP}},
#endif /* __NR_execveat */
/* FS - two paths */
/* FS - two paths */
#ifdef __NR_rename
{__NR_rename, "rename", "rename", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_rename, "rename", "rename", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_rename */
#ifdef __NR_link
{__NR_link, "link", "link", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_link, "link", "link", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_link */
#ifdef __NR_symlink
{__NR_symlink, "symlink", "symlink", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
{__NR_symlink, "symlink", "symlink", SyscallCategory::FS,
{A::PATH, A::PATH, A::SKIP, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_symlink */
/* FS - two dirfd+path pairs */
/* FS - two dirfd+path pairs */
#ifdef __NR_renameat
{__NR_renameat, "renameat", "renameat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
{__NR_renameat, "renameat", "renameat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
#endif /* __NR_renameat */
#ifdef __NR_renameat2
{__NR_renameat2, "renameat2", "renameat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
{__NR_renameat2, "renameat2", "renameat2", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
#endif /* __NR_renameat2 */
#ifdef __NR_linkat
{__NR_linkat, "linkat", "linkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
{__NR_linkat, "linkat", "linkat", SyscallCategory::FS,
{A::DIRFD, A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP}},
#endif /* __NR_linkat */
/* symlinkat: arg0=target(CWD), arg1=newdirfd, arg2=newpath */
/* symlinkat: arg0=target(CWD), arg1=newdirfd, arg2=newpath */
#ifdef __NR_symlinkat
{__NR_symlinkat, "symlinkat", "symlinkat", SyscallCategory::FS,
{A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP}},
{__NR_symlinkat, "symlinkat", "symlinkat", SyscallCategory::FS,
{A::PATH, A::DIRFD, A::PATH, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_symlinkat */
/* NET - sockaddr-based */
/* NET - sockaddr-based */
#ifdef __NR_connect
{__NR_connect, "connect", "connect", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
{__NR_connect, "connect", "connect", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_connect */
#ifdef __NR_bind
{__NR_bind, "bind", "bind", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
{__NR_bind, "bind", "bind", SyscallCategory::NET,
{A::FD, A::SADDR, A::ALEN, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_bind */
#ifdef __NR_sendto
{__NR_sendto, "sendto", "sendto", SyscallCategory::NET,
{A::FD, A::SKIP, A::SKIP, A::IFLAGS, A::SADDR, A::ALEN}},
{__NR_sendto, "sendto", "sendto", SyscallCategory::NET,
{A::FD, A::SKIP, A::SKIP, A::IFLAGS, A::SADDR, A::ALEN}},
#endif /* __NR_sendto */
/* NET - socket creation */
/* NET - socket creation */
#ifdef __NR_socket
{__NR_socket, "socket", "socket", SyscallCategory::NET,
{A::DOMAIN, A::STYPE, A::PROTO, A::SKIP, A::SKIP, A::SKIP}},
{__NR_socket, "socket", "socket", SyscallCategory::NET,
{A::DOMAIN, A::STYPE, A::PROTO, A::SKIP, A::SKIP, A::SKIP}},
#endif /* __NR_socket */
};

267
unotify/unotify.cc
View File

@@ -1,21 +1,32 @@
/*
* Seccomp Unotify subsystem for nsjail.
* This module allows nsjail to observe and log syscalls made by the sandboxed
* process using the SECCOMP_USER_NOTIF feature. It runs a background thread
* that reads notifications, decodes arguments, and aggregates statistics.
* -----------------------------------------
*
* This module allows nsjail to observe and log syscalls made by the
* sandboxed process using the SECCOMP_USER_NOTIF feature. It registers
* the unotify fd with the per-child epoll loop and processes
* notifications in-line (no dedicated thread).
*
* Ownership: start() only takes ownership of the passed fd on success.
* On success, the fd is managed by the event loop (freed via unotify::stop()).
* On failure, the fd is untouched and remains the caller's responsibility to close.
*/
#include "unotify.h"
#include "unotify/unotify.h"
#include <fcntl.h>
#include <linux/seccomp.h>
#include <poll.h>
#include <stdlib.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <sys/types.h>
#include <sys/syscall.h>
#include <time.h>
#include <unistd.h>
#include <thread>
#include "logs.h"
#include "missing_defs.h"
#include "monitor.h"
#include "unotify/record.h"
#include "unotify/stats.h"
#include "unotify/syscall.h"
@@ -23,104 +34,172 @@
namespace unotify {
#ifndef SECCOMP_IOCTL_NOTIF_RECV
#define SECCOMP_IOCTL_NOTIF_RECV SECCOMP_IOWR(0, struct seccomp_notif)
#define SECCOMP_IOCTL_NOTIF_SEND SECCOMP_IOWR(1, struct seccomp_notif_resp)
#define SECCOMP_IOCTL_NOTIF_ID_VALID SECCOMP_IOWR(2, __u64)
#endif
#ifndef SECCOMP_USER_NOTIF_FLAG_CONTINUE
#define SECCOMP_USER_NOTIF_FLAG_CONTINUE (1UL << 0)
#endif
static int unotif_fd = -1;
static std::thread* worker_thread = nullptr;
/*
* Checks if the target process is still alive and the notification ID is valid.
* Returns true if valid, false otherwise.
*/
static bool isTargetAlive(int fd, __u64 last_id) {
return ioctl(fd, SECCOMP_IOCTL_NOTIF_ID_VALID, &last_id) == 0;
}
static void threadMain() {
LOG_I("Started unotify loop");
/*
* When last_id is 0 (no notification received yet), NOTIF_ID_VALID
* will always fail since ID 0 is never valid. This correctly exits
* the loop when NOTIF_RECV has never succeeded (child died before
* making any traced syscall).
*/
__u64 last_id = 0;
while (true) {
struct pollfd pfd = {.fd = unotif_fd, .events = POLLIN, .revents = 0};
int ret = poll(&pfd, 1, -1);
if (ret == -1) {
if (errno == EINTR) continue;
PLOG_E("poll failed");
if (!isTargetAlive(unotif_fd, last_id)) break;
continue;
}
struct seccomp_notif req = {};
if (ioctl(unotif_fd, SECCOMP_IOCTL_NOTIF_RECV, &req) == -1) {
if (errno == EINTR) continue;
PLOG_D("SECCOMP_IOCTL_NOTIF_RECV");
if (!isTargetAlive(unotif_fd, last_id)) break;
continue;
}
last_id = req.id;
LOG_D("Received seccomp notification for syscall %d", req.data.nr);
SyscallRecord rec;
parseSyscall(&req, &rec);
addStat(rec);
if (!isTargetAlive(unotif_fd, req.id)) {
break;
}
struct seccomp_notif_resp resp = {};
resp.id = req.id;
resp.flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE;
if (ioctl(unotif_fd, SECCOMP_IOCTL_NOTIF_SEND, &resp) == -1) {
if (errno != ENOENT) {
PLOG_E("SECCOMP_IOCTL_NOTIF_SEND failed");
}
if (!isTargetAlive(unotif_fd, req.id)) break;
}
}
}
bool start(nsj_t* nsj, int fd) {
if (worker_thread) {
LOG_W("unotify::start() called while already running. "
"Concurrent tracing in LISTEN mode is not yet supported. "
"Closing notification fd for this process.");
close(fd);
return true;
}
unotif_fd = fd;
worker_thread = new std::thread(threadMain);
return true;
return TEMP_FAILURE_RETRY(ioctl(fd, SECCOMP_IOCTL_NOTIF_ID_VALID, &last_id)) == 0;
}
/*
* Called after killAndReapAll(). The worker thread exits its loop when
* isTargetAlive() (SECCOMP_IOCTL_NOTIF_ID_VALID) reports the target is dead.
* Per-fd context holding the notification fd and kernel-reported structure
* sizes for seccomp_notif / seccomp_notif_resp.
*/
void stop(nsj_t* nsj) {
if (worker_thread) {
worker_thread->join();
delete worker_thread;
worker_thread = nullptr;
struct unotifyCtx_t {
int fd = -1;
uint16_t req_size;
uint16_t resp_size;
uint8_t* req_buf = nullptr;
uint8_t* resp_buf = nullptr;
};
static thread_local unotifyCtx_t current_ctx;
static void closeAndUnregister(int fd) {
if (fd >= 0) {
monitor::removeFd(fd);
close(fd);
current_ctx.fd = -1;
}
if (unotif_fd != -1) {
close(unotif_fd);
unotif_fd = -1;
}
void stop() {
closeAndUnregister(current_ctx.fd);
free(current_ctx.req_buf);
free(current_ctx.resp_buf);
current_ctx.req_buf = nullptr;
current_ctx.resp_buf = nullptr;
}
/*
* Epoll callback that processes pending seccomp notifications from the
* kernel. For each notification, it records statistics and sends a CONTINUE
* response so the traced syscall proceeds in the child.
*
* We process one notification per callback to avoid event starvation,
* yielding control back to the event loop.
*/
static void unotifyCb(int fd, uint32_t events, void* /* data */) {
if (events & (EPOLLHUP | EPOLLERR)) {
LOG_D("unotif_fd=%d hung up or error, removing from epoll", fd);
closeAndUnregister(fd);
return;
}
printStats(nsj);
struct seccomp_notif* req = reinterpret_cast<struct seccomp_notif*>(current_ctx.req_buf);
struct seccomp_notif_resp* resp =
reinterpret_cast<struct seccomp_notif_resp*>(current_ctx.resp_buf);
memset(req, 0, current_ctx.req_size);
if (TEMP_FAILURE_RETRY(ioctl(fd, SECCOMP_IOCTL_NOTIF_RECV, req)) == -1) {
/* EAGAIN/EWOULDBLOCK = no more pending */
if (errno == EAGAIN || errno == EWOULDBLOCK) {
return;
}
if (errno == ENOENT) {
LOG_D("unotif_fd=%d returned ENOENT, child likely gone. Removing "
"from epoll.",
fd);
closeAndUnregister(fd);
return;
}
PLOG_W("SECCOMP_IOCTL_NOTIF_RECV failed unexpectedly");
closeAndUnregister(fd);
return;
}
SyscallRecord rec;
LOG_D("unotifyCb: before parseSyscall, nr=%d", req->data.nr);
parseSyscall(req, &rec);
LOG_D("unotifyCb: after parseSyscall");
LOG_D("unotify: syscall=%s id=%llx", rec.name.c_str(), (unsigned long long)req->id);
addStat(rec);
if (!isTargetAlive(fd, req->id)) {
return;
}
memset(resp, 0, current_ctx.resp_size);
resp->id = req->id;
if (rec.name == "connect" && rec.res.has_net &&
(rec.res.net_type == Stat_NetResource_Type_IPV4 ||
rec.res.net_type == Stat_NetResource_Type_IPV6)) {
resp->flags = 0;
resp->error = -ECONNREFUSED;
resp->val = -1;
LOG_D("unotify: failing network connect with ECONNREFUSED");
} else {
resp->flags = SECCOMP_USER_NOTIF_FLAG_CONTINUE;
}
if (TEMP_FAILURE_RETRY(ioctl(fd, SECCOMP_IOCTL_NOTIF_SEND, resp)) == -1) {
if (errno != ENOENT) {
PLOG_W("SECCOMP_IOCTL_NOTIF_SEND");
}
}
}
/*
* Initializes the unotify monitoring for one child process.
*
* Queries SECCOMP_GET_NOTIF_SIZES from the kernel to learn the correct
* allocation sizes, sets the fd non-blocking, and registers it with the
* epoll loop.
*
* Returns true if the fd was successfully absorbed by the loop. On failure,
* the fd is untouched and remains the caller's responsibility to close.
*/
bool start(nsj_t* nsj, int fd) {
thread_local struct seccomp_notif_sizes sizes = {0, 0, 0};
if (sizes.seccomp_notif == 0) {
if (util::syscall(__NR_seccomp, SECCOMP_GET_NOTIF_SIZES, 0, (uintptr_t)&sizes) ==
-1) {
PLOG_W("seccomp(SECCOMP_GET_NOTIF_SIZES)");
return false;
}
}
current_ctx.fd = fd;
current_ctx.req_size = sizes.seccomp_notif;
current_ctx.resp_size = sizes.seccomp_notif_resp;
current_ctx.req_buf = static_cast<uint8_t*>(malloc(sizes.seccomp_notif));
current_ctx.resp_buf = static_cast<uint8_t*>(malloc(sizes.seccomp_notif_resp));
if (!current_ctx.req_buf || !current_ctx.resp_buf) {
LOG_E("Failed to allocate unotify buffers");
free(current_ctx.req_buf);
free(current_ctx.resp_buf);
current_ctx.req_buf = nullptr;
current_ctx.resp_buf = nullptr;
return false;
}
if (!util::setNonBlock(fd)) {
free(current_ctx.req_buf);
free(current_ctx.resp_buf);
current_ctx.req_buf = nullptr;
current_ctx.resp_buf = nullptr;
current_ctx.fd = -1;
return false;
}
if (!monitor::addFd(fd, EPOLLIN, unotifyCb, nullptr)) {
PLOG_W("monitor::addFd for unotify failed");
free(current_ctx.req_buf);
free(current_ctx.resp_buf);
current_ctx.req_buf = nullptr;
current_ctx.resp_buf = nullptr;
current_ctx.fd = -1;
return false;
}
return true;
}
} // namespace unotify

3
unotify/unotify.h
View File

@@ -6,8 +6,7 @@
namespace unotify {
bool start(nsj_t* nsj, int unotif_fd);
void stop(nsj_t* nsj);
/* Note: printStats is in unotify/stats.h, exposed to nsjail.cc */
void stop(void);
} // namespace unotify

27
user.cc
View File

@@ -73,7 +73,7 @@ static bool setResGid(gid_t gid) {
PLOG_W("setresgid(%d)", gid);
return false;
}
#endif /* defined(__NR_setresuid32) */
#endif /* defined(__NR_setresgid32) */
return true;
}
@@ -111,11 +111,10 @@ static bool setGroupsDeny(nsj_t* nsj, pid_t pid) {
return true;
}
char fname[PATH_MAX];
snprintf(fname, sizeof(fname), "/proc/%d/setgroups", pid);
std::string fname = "/proc/" + std::to_string(pid) + "/setgroups";
const char* const denystr = "deny";
if (!util::writeBufToFile(fname, denystr, strlen(denystr), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname, denystr);
if (!util::writeBufToFile(fname.c_str(), denystr, strlen(denystr), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname.c_str(), denystr);
return false;
}
return true;
@@ -138,11 +137,10 @@ static bool uidMapSelf(nsj_t* nsj, pid_t pid) {
return true;
}
char fname[PATH_MAX];
snprintf(fname, sizeof(fname), "/proc/%d/uid_map", pid);
LOG_D("Writing '%s' to '%s'", map.c_str(), fname);
if (!util::writeBufToFile(fname, map.data(), map.length(), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname, map.c_str());
std::string fname = "/proc/" + std::to_string(pid) + "/uid_map";
LOG_D("Writing '%s' to '%s'", map.c_str(), fname.c_str());
if (!util::writeBufToFile(fname.c_str(), map.data(), map.length(), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname.c_str(), map.c_str());
return false;
}
@@ -166,11 +164,10 @@ static bool gidMapSelf(nsj_t* nsj, pid_t pid) {
return true;
}
char fname[PATH_MAX];
snprintf(fname, sizeof(fname), "/proc/%d/gid_map", pid);
LOG_D("Writing '%s' to '%s'", map.c_str(), fname);
if (!util::writeBufToFile(fname, map.data(), map.length(), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname, map.c_str());
std::string fname = "/proc/" + std::to_string(pid) + "/gid_map";
LOG_D("Writing '%s' to '%s'", map.c_str(), fname.c_str());
if (!util::writeBufToFile(fname.c_str(), map.data(), map.length(), O_WRONLY | O_CLOEXEC)) {
LOG_E("util::writeBufToFile('%s', '%s') failed", fname.c_str(), map.c_str());
return false;
}

12
util.cc
View File

@@ -165,18 +165,6 @@ bool recvMsg(int sock, uint32_t* msg_val, int* fd) {
return true;
}
bool sendFd(int sock, int fd) {
return sendMsg(sock, 0, fd);
}
int recvFd(int sock) {
int fd = -1;
if (recvMsg(sock, nullptr, &fd)) {
return fd;
}
return -1;
}
bool readFromFileToStr(const char* fname, std::string* str) {
int fd = TEMP_FAILURE_RETRY(open(fname, O_RDONLY | O_CLOEXEC));
if (fd == -1) {

5
util.h
View File

@@ -33,6 +33,7 @@
#include <string>
#include <vector>
#include "missing_defs.h"
#include "nsjail.h"
#define RETURN_ON_FAILURE(expr) \
@@ -53,10 +54,6 @@ bool writeToFd(int fd, const void* buf, size_t len);
bool sendMsg(int sock, uint32_t msg, int fd = -1);
bool recvMsg(int sock, uint32_t* msg, int* fd = nullptr);
bool sendFd(int sock, int fd);
int recvFd(int sock);
bool writeBufToFile(
const char* filename, const void* buf, size_t len, int open_flags, bool log_errors = true);
bool createDirRecursively(const char* dir);