Fix NSTUN parent lifecycle cleanup

net.cc

@@ -417,6 +417,15 @@ static bool spawnPasta(nsj_t* nsj, int pid) {
 }
 
 bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
+	bool nstun_started = false;
+	bool init_success = false;
+	defer {
+		if (!init_success && nstun_started && nsj->pids[pid].nstun != nullptr) {
+			nstun_destroy_parent(nsj->pids[pid].nstun);
+			nsj->pids[pid].nstun = nullptr;
+		}
+	};
+
 	if (nsj->njc.has_user_net()) {
 		if (!nsj->njc.clone_newnet()) {
 			LOG_E("Support for User-Mode Networking requested but CLONE_NEWNET "
@@ -424,10 +433,11 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
 			return false;
 		}
 		if (nsj->njc.user_net().backend() == nsjail::NsJailConfig_UserNet_Backend_NSTUN) {
-			if (!nstun_init_parent(pipefd, nsj)) {
+			if (!nstun_init_parent(pipefd, nsj, &nsj->pids[pid].nstun)) {
 				LOG_E("nstun_init_parent() failed");
 				return false;
 			}
+			nstun_started = true;
 		} else if (nsj->njc.user_net().backend() ==
 			       nsjail::NsJailConfig_UserNet_Backend_PASTA &&
 			   nsj->njc.user_net().has_pasta()) {
@@ -437,6 +447,7 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
 		}
 	}
 	if (!nsj->njc.clone_newnet()) {
+		init_success = true;
 		return true;
 	}
 #ifdef HAVE_LIBNL3
@@ -473,6 +484,7 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
 		return false;
 	}
 
+	init_success = true;
 	return true;
 #else
 	if (!nsj->njc.iface_own().empty() || !nsj->njc.macvlan_iface().empty()) {
@@ -480,6 +492,7 @@ bool initParent(nsj_t* nsj, pid_t pid, int pipefd) {
 		      "was built without libnl3 support");
 		return false;
 	}
+	init_success = true;
 	return true;
 #endif
 }

nsjail.h

@@ -38,6 +38,8 @@
 
 #include "config.pb.h"
 
+struct nstun_context_handle;
+
 static const int nssigs[] = {
     SIGINT,
     SIGQUIT,
@@ -59,6 +61,7 @@ struct pids_t {
 	int pid_syscall_fd;
 	pid_t pasta_pid;
 	pthread_t monitor_tid;
+	nstun_context_handle* nstun = nullptr;
 };
 
 struct idmap_t {

nstun/core.h

@@ -149,6 +149,7 @@ struct IcmpFlow : public Flow {
 struct Context {
 	int epoll_fd;
 	int tap_fd;
+	int stop_fd = -1;
 	struct nsj_t* nsj;
 
 	/* IP addresses in network byte order */

nstun/nstun.cc

@@ -11,6 +11,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <sys/epoll.h>
+#include <sys/eventfd.h>
 #include <sys/ioctl.h>
 #include <sys/socket.h>
 #include <sys/wait.h>
@@ -31,6 +32,11 @@
 #include "udp.h"
 #include "util.h"
 
+struct nstun_context_handle {
+	nstun::Context* context = nullptr;
+	std::thread worker;
+};
+
 namespace nstun {
 
 Context::~Context() {
@@ -91,11 +97,24 @@ static void networkLoop(Context* ctx) {
 	LOG_D("nstun network loop started on tap_fd=%d", ctx->tap_fd);
 
 	defer {
-		close(ctx->tap_fd);
-		close(ctx->epoll_fd);
-		delete ctx;
+		if (ctx->tap_fd != -1) {
+			close(ctx->tap_fd);
+			ctx->tap_fd = -1;
+		}
+		if (ctx->epoll_fd != -1) {
+			close(ctx->epoll_fd);
+			ctx->epoll_fd = -1;
+		}
+		/* The owner closes stop_fd after joining. Keeping it open prevents
+		 * reuse if the loop exits naturally before child reap. */
 	};
 
+	struct epoll_event stop_ev = {.events = EPOLLIN, .data = {.fd = ctx->stop_fd}};
+	if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, ctx->stop_fd, &stop_ev) == -1) {
+		PLOG_E("epoll_ctl(EPOLL_CTL_ADD, stop_fd)");
+		return;
+	}
+
 	struct epoll_event ev = {.events = EPOLLIN, .data = {.fd = ctx->tap_fd}};
 	if (epoll_ctl(ctx->epoll_fd, EPOLL_CTL_ADD, ctx->tap_fd, &ev) == -1) {
 		PLOG_E("epoll_ctl(EPOLL_CTL_ADD, tap_fd)");
@@ -125,7 +144,11 @@ static void networkLoop(Context* ctx) {
 		for (int i = 0; i < nfds; ++i) {
 			int fd = events[i].data.fd;
 
-			if (fd == ctx->tap_fd) {
+			if (fd == ctx->stop_fd) {
+				uint64_t value;
+				(void)TEMP_FAILURE_RETRY(read(ctx->stop_fd, &value, sizeof(value)));
+				return;
+			} else if (fd == ctx->tap_fd) {
 				ssize_t n = TEMP_FAILURE_RETRY(
 				    read(ctx->tap_fd, buf.get(), TUN_FRAME_BUF_SIZE));
 				if (n <= 0) {
@@ -187,7 +210,14 @@ bool nstun_init_child(int sock, nsj_t* nsj) {
 	return true;
 }
 
-bool nstun_init_parent(int sock, nsj_t* nsj) {
+bool nstun_init_parent(int sock, nsj_t* nsj, struct nstun_context_handle** out_handle) {
+	if (out_handle == nullptr) {
+		LOG_E("nstun_init_parent() requires an output handle");
+		return false;
+	}
+	*out_handle = nullptr;
+	auto handle = std::make_unique<nstun_context_handle>();
+
 	int tap_fd = util::recvFd(sock);
 	if (tap_fd < 0) {
 		LOG_E("Failed to receive TAP fd from child");
@@ -288,13 +318,31 @@ bool nstun_init_parent(int sock, nsj_t* nsj) {
 		close(ctx->tap_fd);
 		return false;
 	}
+	ctx->stop_fd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
+	if (ctx->stop_fd == -1) {
+		PLOG_E("eventfd(stop_fd)");
+		close(ctx->epoll_fd);
+		close(ctx->tap_fd);
+		return false;
+	}
 
 	auto cleanup_and_fail = [&ctx]() -> bool {
 		for (auto& [fd, _] : ctx->host_listener_fd_to_rule) {
 			close(fd);
 		}
-		close(ctx->epoll_fd);
-		close(ctx->tap_fd);
+		ctx->host_listener_fd_to_rule.clear();
+		if (ctx->stop_fd != -1) {
+			close(ctx->stop_fd);
+			ctx->stop_fd = -1;
+		}
+		if (ctx->epoll_fd != -1) {
+			close(ctx->epoll_fd);
+			ctx->epoll_fd = -1;
+		}
+		if (ctx->tap_fd != -1) {
+			close(ctx->tap_fd);
+			ctx->tap_fd = -1;
+		}
 		return false;
 	};
 
@@ -496,8 +544,34 @@ bool nstun_init_parent(int sock, nsj_t* nsj) {
 	}
 
 	/* Spawn network loop thread */
-	std::thread t(nstun::networkLoop, ctx.release());
-	t.detach();
+	handle->context = ctx.get();
+	handle->worker = std::thread(nstun::networkLoop, handle->context);
+	ctx.release();
+	*out_handle = handle.release();
 
 	return true;
 }
+
+void nstun_destroy_parent(struct nstun_context_handle* handle) {
+	if (handle == nullptr) {
+		return;
+	}
+
+	if (handle->context != nullptr && handle->context->stop_fd != -1) {
+		const uint64_t one = 1;
+		if (TEMP_FAILURE_RETRY(
+			write(handle->context->stop_fd, &one, sizeof(one))) == -1 && errno != EPIPE) {
+			PLOG_W("write(stop_fd)");
+		}
+	}
+	if (handle->worker.joinable()) {
+		handle->worker.join();
+	}
+	if (handle->context != nullptr && handle->context->stop_fd != -1) {
+		close(handle->context->stop_fd);
+		handle->context->stop_fd = -1;
+	}
+	delete handle->context;
+	handle->context = nullptr;
+	delete handle;
+}

nstun/nstun.h

@@ -49,12 +49,15 @@ typedef struct {
 	uint16_t redirect_port;
 } nstun_rule_t;
 struct nsj_t;
+struct nstun_context_handle;
 
 bool nstun_init_child(int sock, struct nsj_t* nsj);
-bool nstun_init_parent(int sock, struct nsj_t* nsj);
+bool nstun_init_parent(
+	int sock, struct nsj_t* nsj, struct nstun_context_handle** out_handle);
+void nstun_destroy_parent(struct nstun_context_handle* handle);
 
 #ifdef __cplusplus
 }
 #endif
 
-#endif /* NSTUN_H_ */
+#endif /* NSTUN_H_ */

subproc.cc

@@ -283,7 +283,11 @@ static void removeProc(nsj_t* nsj, pid_t pid) {
 		return;
 	}
 
-	const auto& p = nsj->pids[pid];
+	auto& p = nsj->pids[pid];
+	if (p.nstun != nullptr) {
+		nstun_destroy_parent(p.nstun);
+		p.nstun = nullptr;
+	}
 	if (p.pasta_pid > 0) {
 		LOG_D("Killing pasta pid=%d", p.pasta_pid);
 		kill(p.pasta_pid, SIGKILL);

tests/nstun-lifecycle-compile-only.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+set -eu
+
+# Build-only gate for a Linux worker with nsjail's documented protobuf/libnl
+# dependencies. No binary is launched and no namespace, TUN, or network setup
+# is performed by this script.
+make -j2 nstun/nstun.o net.o subproc.o

tests/nstun-listen-lifecycle-authoritative.sh

@@ -0,0 +1,135 @@
+#!/bin/sh
+set -eu
+
+# Linux/root integration fixture. It uses only a local LISTEN socket and the
+# disposable network namespace in the config; it never contacts an external
+# service. Override NSJAIL_BIN, CONFIG, PORT, COUNT, and TIMEOUT_SECONDS.
+NSJAIL_BIN=${NSJAIL_BIN:-./nsjail}
+CONFIG=${CONFIG:-tests/nstun-listen-lifecycle.cfg}
+PORT=${PORT:-38080}
+COUNT=${COUNT:-10}
+TIMEOUT_SECONDS=${TIMEOUT_SECONDS:-3}
+LOG=${LOG:-${TMPDIR:-/tmp}/nsjail-nstun-lifecycle.parent.log}
+TMP_CFG=$(mktemp "${TMPDIR:-/tmp}/nstun-config.XXXXXX")
+RAW=${RAW:-${TMPDIR:-/tmp}/nsjail-nstun-lifecycle.raw}
+mkdir -p "$RAW"
+
+cleanup() {
+	set +e
+	if [ "${NSJAIL_PID:-}" != "" ] && kill -0 "$NSJAIL_PID" 2>/dev/null; then
+		kill -TERM "$NSJAIL_PID" 2>/dev/null
+		for _ in $(seq 1 40); do
+			kill -0 "$NSJAIL_PID" 2>/dev/null || break
+			sleep 0.05
+		done
+		kill -KILL "$NSJAIL_PID" 2>/dev/null
+		wait "$NSJAIL_PID" 2>/dev/null
+	fi
+	rm -f "$TMP_CFG"
+}
+trap cleanup EXIT INT TERM
+
+sed "s/^port: .*/port: ${PORT}/" "$CONFIG" >"$TMP_CFG"
+: >"$LOG"
+"$NSJAIL_BIN" -C "$TMP_CFG" >"$LOG" 2>&1 &
+NSJAIL_PID=$!
+
+for _ in $(seq 1 100); do
+	if ss -H -ltn 2>/dev/null | awk -v p=":${PORT}" '$4 ~ p"$" {ok=1} END {exit !ok}'; then
+		break
+	fi
+	sleep 0.05
+done
+kill -0 "$NSJAIL_PID" 2>/dev/null || {
+	echo "nsjail exited before readiness" >&2
+	exit 1
+}
+
+counts() {
+	pid=$1
+	tasks=$(find "/proc/$pid/task" -mindepth 1 -maxdepth 1 -type d | wc -l)
+	fds=$(find "/proc/$pid/fd" -mindepth 1 -maxdepth 1 -type l | wc -l)
+	threads=$(awk '/^Threads:/{print $2}' "/proc/$pid/status")
+	printf '%s %s %s\n' "$tasks" "$fds" "$threads"
+}
+
+targets() {
+	pid=$1
+	for fd in /proc/$pid/fd/*; do
+		[ -e "$fd" ] || continue
+		readlink "$fd" 2>/dev/null || true
+	done | sort
+}
+
+snapshot() {
+	label=$1
+	counts "$NSJAIL_PID" >"$RAW/$label.counts"
+	targets "$NSJAIL_PID" >"$RAW/$label.targets"
+	printf '%s\t%s\n' "$label" "$(cat "$RAW/$label.counts")" >&2
+}
+
+client_once() {
+	python3 - "$PORT" <<'PY'
+import socket, sys
+s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+s.settimeout(2)
+try:
+    s.connect(("127.0.0.1", int(sys.argv[1])))
+    s.sendall(b"nstun-lifecycle\n")
+    try:
+        s.recv(1)
+    except OSError:
+        pass
+finally:
+    s.close()
+PY
+}
+
+snapshot idle
+baseline=$(cat "$RAW/idle.counts")
+cp "$RAW/idle.counts" "$RAW/baseline.counts"
+cp "$RAW/idle.targets" "$RAW/baseline.targets"
+echo "baseline=$baseline"
+
+for i in $(seq 1 "$COUNT"); do
+	client_once || true
+	deadline=$(( $(date +%s) + TIMEOUT_SECONDS ))
+	while [ "$(grep -Ec 'pid=.*(exited|terminated with signal)' "$LOG" || true)" -lt "$i" ]; do
+		[ "$(date +%s)" -lt "$deadline" ] || {
+			echo "child $i was not reaped" >&2
+			exit 1
+		}
+		sleep 0.05
+	done
+	# The exit log is emitted before removeProc() performs NSTUN teardown.
+	# Poll the independent parent readbacks until the join/close completes.
+	while :; do
+		counts "$NSJAIL_PID" >"$RAW/post-$i.candidate.counts"
+		targets "$NSJAIL_PID" >"$RAW/post-$i.candidate.targets"
+		if cmp -s "$RAW/baseline.counts" "$RAW/post-$i.candidate.counts" &&
+			cmp -s "$RAW/baseline.targets" "$RAW/post-$i.candidate.targets"; then
+			cp "$RAW/post-$i.candidate.counts" "$RAW/post-$i.counts"
+			cp "$RAW/post-$i.candidate.targets" "$RAW/post-$i.targets"
+			break
+		fi
+		[ "$(date +%s)" -lt "$deadline" ] || {
+			echo "parent resources did not return to baseline after child $i" >&2
+			exit 1
+		}
+		sleep 0.05
+	done
+	current=$(cat "$RAW/post-$i.counts")
+	echo "post-$i=$current"
+	[ "$current" = "$baseline" ] || {
+		echo "resource count changed after child $i: $baseline -> $current" >&2
+		exit 1
+	}
+	diff -u "$RAW/baseline.targets" "$RAW/post-$i.targets"
+done
+
+if grep -Eq '^(/net/tun|anon_inode:\[eventpoll\])$' "$RAW/baseline.targets" "$RAW/post-$COUNT.targets"; then
+	echo "unexpected NSTUN-owned descriptor remained in the parent" >&2
+	exit 1
+fi
+
+echo "PASS: ${COUNT} NSTUN LISTEN children returned parent resources to baseline"

tests/nstun-listen-lifecycle.cfg

@@ -0,0 +1,46 @@
+name: "nstun-listen-lifecycle"
+mode: LISTEN
+bindhost: "127.0.0.1"
+port: 38080
+max_conns: 1
+hostname: "nstun-listen-lifecycle"
+
+clone_newnet: true
+clone_newuser: true
+clone_newns: true
+clone_newpid: true
+clone_newipc: true
+clone_newuts: true
+
+uidmap {
+	inside_id: "0"
+	outside_id: ""
+	count: 1
+}
+
+gidmap {
+	inside_id: "0"
+	outside_id: ""
+	count: 1
+}
+
+mount_proc: true
+
+mount {
+	src: "/"
+	dst: "/"
+	is_bind: true
+	nosuid: true
+	nodev: false
+	noexec: false
+}
+
+user_net {
+	backend: NSTUN
+}
+
+exec_bin {
+	path: "/bin/sh"
+	arg: "-c"
+	arg: "sleep 0.1"
+}