🏅 Sync 2026-11-10 05:39:45

shunt/Makefile

@@ -5,8 +5,8 @@
 include $(TOPDIR)/rules.mk
 
 PKG_NAME:=shunt
-PKG_VERSION:=0.4.0
-PKG_RELEASE:=12
+PKG_VERSION:=0.4.2
+PKG_RELEASE:=13
 PKG_LICENSE:=GPL-3.0-or-later
 PKG_LICENSE_FILES:=
 PKG_MAINTAINER:=Dirk Brenken <dev@brenken.org>

shunt/README.md

@@ -437,7 +437,7 @@ Note that domain precedence is resolved *before* this, at the matcher: the most
 
 `fallback 'main'` (default) renders no default route into the policy table, so when the policy interface is down the table is empty and marked traffic falls through to `main` - the normal uplink. Traffic keeps flowing, unpolicied.
 
-`fallback 'block'` adds a `prohibit` default at metric 9999 to the policy table. While the interface is up its own default has the lower metric and wins; when the interface drops, the kernel withdraws that route and the prohibit route catches everything, answering it with ICMP admin-prohibited - a connection attempt fails at once instead of on a timeout, measured. That is the killswitch: traffic belonging to the policy stops rather than leaking out of the wrong interface. It stops at the destinations the policy actually owns - see [keep_local](#local-traffic-keep_local) for what a marked packet to one of your own networks does, and how to make the killswitch hermetic.
+`fallback 'block'` adds a `prohibit` default at metric 9999 to the policy table. While the interface is up its own default has the lower metric and wins; when the interface drops, the kernel withdraws that route and the prohibit route catches everything, answering it with ICMP admin-prohibited. The kernel rate-limits those ICMP errors per client, though, and a client that keeps retrying flows into the blocked policy uses up its budget, so its next connection attempt would be dropped silently and time out - measured. While every interface of the policy is down or dead, its mark chain therefore resets tcp itself, which is not rate-limited: a tcp connection fails at once, whatever else the client sends; everything else still gets the ICMP answer. That is the killswitch: traffic belonging to the policy stops rather than leaking out of the wrong interface. It stops at the destinations the policy actually owns - see [keep_local](#local-traffic-keep_local) for what a marked packet to one of your own networks does, and how to make the killswitch hermetic.
 
 <a id="health-tracking-track_ip"></a>
 ### Health tracking: track_ip
@@ -453,7 +453,9 @@ config global
 
 Every interface a `route` policy uses is then probed every 5 seconds, each target once per round. A round counts as answered when any target replies, so one unreachable target, or a tunnel without IPv6, does not take the interface down. Three unanswered rounds in a row make the interface dead, three answered ones bring it back - 15 to 20 seconds either way, measured. The timing has no options.
 
-A dead device is handled exactly like one whose link went down: the default route leaves its policies' tables and the [fallback](#fallback-main-or-block) takes over. With `block` new connections are rejected at once; with `main` they take the normal uplink. Connections that were running into the dead interface are moved by that too and reset by the kernel, which they would not have survived anyway. Connections opened through `main` during the outage are left unmarked, so they stay on the normal uplink when the interface comes back instead of being torn down - only new connections return to it, measured. `tproxy` and `bypass` policies are not tracked.
+**List targets of both families.** A target only tests the path of its own family, so the list decides what "alive" means. With IPv6 targets alone an interface is judged by its IPv6 path only: a tunnel without IPv6 is never seen alive, and a dual-stack tunnel whose IPv6 lags behind after boot or fails on its own counts as dead while IPv4 works fine - the same holds the other way round with IPv4 targets alone. Whenever the policy interfaces carry both families, or differ in which ones they carry, list at least one IPv4 and one IPv6 target, as in the example above.
+
+A dead device is handled exactly like one whose link went down: the default route leaves its policies' tables and the [fallback](#fallback-main-or-block) takes over. With `block` new connections are rejected - tcp with a reset, at once; with `main` they take the normal uplink. Connections that were running into the dead interface are moved by that too and reset by the kernel, which they would not have survived anyway. Connections opened through `main` during the outage are left unmarked, so they stay on the normal uplink when the interface comes back instead of being torn down - only new connections return to it, measured. `tproxy` and `bypass` policies are not tracked.
 
 The probes leave from the router through the policy device itself: from a raw socket bound to the device, with a mark of its own that leads to a probe table holding the same default route as the device's policies. That table is never withdrawn, so a dead device keeps being probed and is noticed when it recovers, and the binding keeps a probe from ever leaving through another uplink and reporting the wrong interface alive. Two things follow from that path. The replies arrive on the policy device from an address the main table routes elsewhere, so `rp_filter` must be loose there - as it must for shunt anyway, see [Which tunnels work](#which-tunnels-work). And on WireGuard the targets must lie within the peer's `AllowedIPs`, which a full tunnel's `0.0.0.0/0` and `::/0` cover.
 
@@ -706,28 +708,32 @@ routing table                        8000 + <index>, for tproxy: local default d
                                      with fallback 'block': prohibit default metric 9999
                                      8500 + <n>, a tracked interface's probe table
 chain mk_<policy>                    the mark statement of the interface in use, for a policy
-                                     with several interfaces, or with track_ip and fallback 'main'
+                                     with several interfaces, with fallback 'block' (plus a tcp
+                                     reset while all are dead), or with track_ip and fallback 'main'
 /etc/iproute2/rt_tables.d/shunt.conf the table name mapping, for `ip route show` only
+/var/run/shunt.learned               learned addresses across a restart or reload, deleted on start
 ```
 
 A `bypass` policy is only a rule in the prerouting and output chains: no mark, no table, no ip rule, and it does not count against the 255. A `tproxy` policy has rules in prerouting only, and no keep_local rule. The mark mask is fixed at `0xff000000`, which allows 255 policy interfaces - a policy with several interfaces takes one for each; with `track_ip` each tracked interface takes one more for its probes. The `output` chain is `type route` so the router's own marked traffic is re-routed after the mark is set.
 
 Every set carries per-element counters, so "is this element ever hit" is one look at `nft list set inet shunt <set>` rather than a tcpdump session. The two kinds count different things: nftables tests a rule left to right, so a **client** set counts every packet that matched the selector, whether or not the destination matched afterwards; a **learned** set is the last lookup in the rule, so a hit there means the packet really was marked. A busy client beside learned addresses at zero is a client that has not visited any of the routed domains, not a fault.
 
-**The table can be deleted from outside, and comes back.** `fw4 reload` and `fw4 restart` touch only `table inet fw4`, so shunt's table survives both. `fw4 flush` does not: it walks `nft list tables` and deletes every table it finds, shunt's included - and that is exactly what `/etc/init.d/firewall stop` runs. Nothing in the kernel reports the loss, so the daemon checks that its table is still there once per poll cycle, and immediately whenever a write into a learned set fails. If it is gone, the ruleset is re-applied and the write cache is dropped, so poll and snoop refill the learned sets rather than suppressing addresses the kernel no longer has:
+**The table can be deleted from outside, and comes back.** `fw4 reload` and `fw4 restart` touch only `table inet fw4`, so shunt's table survives both. `fw4 flush` does not: it walks `nft list tables` and deletes every table it finds, shunt's included - and that is exactly what `/etc/init.d/firewall stop` runs. Nothing in the kernel reports the loss, so the daemon checks that its table is still there once per poll cycle, and immediately whenever a write into a learned set fails. If it is gone, the ruleset is re-applied and every learned address the daemon still has on record goes back into its set with the time it has left, on the next write - two seconds at most on an ordinary box. The kernel lost the elements, the daemon did not:
 
 ```
 shunt: ruleset gone from the kernel - re-applying; `fw4 flush`, as run by `/etc/init.d/firewall stop`, deletes every nftables table
-shunt: ruleset re-applied - learned addresses refill from the next poll cycle and from snoop
+shunt: ruleset re-applied - 42 learned address(es) queued to be written back
 ```
 
 The routes and ip rules are not affected by any of this - they are re-asserted every poll cycle anyway. Until the check runs, marked traffic simply is not marked, so it takes the normal uplink; with `fallback 'block'` that means the killswitch is not in force either. `ubus call shunt status` counts the recoveries as `reapplied`.
 
 **Writes are batched, and the interval adapts.** `nft -f` reads the entire ruleset from the kernel before it resolves a single name, so on a box that also runs a tool with very large sets - banIP with 238k elements, measured - one `add element` costs seconds of CPU, and `nft --check` alone costs the same. That is a known bug in nftables (netfilter bugzilla #1735, open since 2024), not something shunt can fix, so observed addresses are collected and applied together by a timer.
 
-The interval follows what the last write actually cost, between 2 and 60 seconds: on an ordinary box a write takes milliseconds and the interval stays at its floor, where the batching is invisible. Where it is expensive the interval grows until nftables takes a bounded share of the machine instead of all of it, at the price of a learned address reaching its set later. Both numbers show up under `debug`.
+The interval follows what the last write actually cost, between 2 and 60 seconds: on an ordinary box a write takes milliseconds and the interval stays at its floor. Where it is expensive the interval grows until nftables takes a bounded share of the machine instead of all of it, at the price of a learned address reaching its set later. Both numbers show up under `debug`.
 
-**A reload wipes learned state.** Applying the configuration destroys and re-creates the table atomically, so the learned sets start empty. poll rewarms them within one interval and snoop refills from live traffic; expect a short window after a restart where domain policies do not apply yet.
+The timer only carries refreshes of addresses already in a set. While the interval is at its floor, an address the sets do not have yet - never seen, or lapsed - is written right after the answer that brought it, together with whatever else is queued, at most once a second; a new address that hits that limit goes out one second later. A client typically opens several connections after one lookup, and without this every one of them in the first two seconds would miss the element. Once the interval has grown, new addresses wait for the timer like everything else, because the write itself is what costs seconds there.
+
+**A reload keeps learned state.** Applying the configuration destroys and re-creates the table atomically, and a reload is a restart of the daemon, so on its way out the daemon writes the learned addresses it holds to `/var/run/shunt.learned` and the next start writes them back with the time they have left. An address comes back only into its own policy, and only if that policy's `domain` and `domain_file` entries are unchanged - the file names, not the file contents, the same as `shunt refresh` - so a policy that now names other domains starts empty instead of inheriting the old ones. The file is read once and deleted, and only a restart or reload uses it: `/etc/init.d/shunt stop` followed by `start` begins empty, which is also the way to drop learned addresses on purpose. It lives in tmpfs, so after a reboot, or when the daemon crashed instead of stopping, the sets start empty too and poll and snoop refill them. The log says how many addresses came over and how many were dropped with a changed or removed policy.
 
 **A flow is routed once, on its first packet.** The mark a `route` policy sets is also written to the flow's conntrack entry, and every later packet of the flow in the original direction takes its mark from there and never reaches the set lookups; flows that got no mark, or a `bypass`, stay settled the same way. That is what makes a set change safe for connections already running: the kernel kills a masqueraded conntrack entry whose output interface changes, so re-marking a live flow would reset it - measured, not assumed. The same rule holds across a reload. Conntrack survives it, so a connection keeps the decision it started with, including the policy index encoded in its mark; if a reload reorders the policies, that index may now belong to a different policy. Existing connections are not re-evaluated against the new configuration, new connections follow it. Reboot, or restart the client's connections, if that matters after a reorder.
 
@@ -829,9 +835,9 @@ These are consequences of the design, stated rather than worked around:
 * **Clients that speak DoH or DoT themselves are invisible to snoop.** poll still covers the names you list explicitly; wildcards do not work for those clients. A *resolver* forwarding upstream over DoT or DoH changes nothing.
 * **Wildcards require snoop.** poll can only resolve names it was given, and `*.example.com` is not a name.
 * **One CDN address serves many domains.** If a policy routes `example.com` and the address behind it also serves a thousand other sites, those sites follow the same policy. This is unsolvable at layer 3 by anything that routes on addresses.
-* **The first connection to a newly seen address takes the old path.** snoop learns from the response the client is reading at that moment, so the client's SYN is usually out before the element reaches the set. That connection stays where it started, by design: a flow's route is decided on its first packet and kept on the conntrack entry, so the switch happens at a connection boundary and never mid-stream - see [What shunt creates on the system](#what-shunt-creates-on-the-system). The next connection to the same host starts on the policy interface. Listing the entry point explicitly closes the gap, because poll warms it before any client asks.
+* **The first connection to a newly seen address takes the old path.** snoop learns from the response the client is reading at that moment, so the client's SYN is usually out before the element reaches the set - a new address is written right away, but that still takes an `nft` run, and a client connects within milliseconds of its answer. Holding the answer back until the set is written would put shunt between client and resolver, which is exactly what snoop avoids. That connection stays where it started, by design: a flow's route is decided on its first packet and kept on the conntrack entry, so the switch happens at a connection boundary and never mid-stream - see [What shunt creates on the system](#what-shunt-creates-on-the-system). The next connection to the same host starts on the policy interface. Listing the entry point explicitly closes the gap, because poll warms it before any client asks.
 * **DNS over TCP is not observed.** Port 53 over TCP needs reassembly, which is out of scope; answers large enough to force TCP are rare in the traffic shunt cares about.
-* **Route and rule application is best effort.** At boot a tunnel interface may not exist yet. A rule over an empty table falls through to `main`, so the failure mode is "policy not applied yet", never "traffic broken". Each distinct reason is one warning line.
+* **Route and rule application is best effort.** At boot a tunnel interface may not exist yet, so its default route cannot be added and the policy's `fallback` applies: with `main` the rule falls through to the main table, with `block` it hits the prohibit route, which needs no interface. The missing routes follow on the interface's ifup event, no restart needed, and the log says so in one info line naming the interfaces. A route that fails for any other reason - or through an interface netifd does not know at all - is a warning, one line per distinct reason.
 * **No interface hotplug.** A device that appears later is picked up on the next `ifup` event or within one poll interval, not immediately.
 * **An outside flush of nftables is repaired, not prevented.** Any tool may delete shunt's table - `fw4 flush` does. shunt re-applies it within one poll interval, or sooner, and until then nothing is marked.
 

shunt/files/shunt.init

@@ -8,6 +8,7 @@ START=95
 USE_PROCD=1
 
 shunt_init="/etc/init.d/shunt"
+shunt_learned="/var/run/shunt.learned"
 
 if [ -z "${IPKG_INSTROOT}" ]; then
 	case "${action}" in
@@ -24,6 +25,13 @@ fi
 
 start_service() {
 	if "${shunt_init}" enabled; then
+		case "${action}" in
+		"restart" | "reload") ;;
+
+		*)
+			rm -f "${shunt_learned}"
+			;;
+		esac
 		procd_open_instance "shunt"
 		procd_set_param command /usr/sbin/shunt run
 		procd_set_param respawn 300 5 3

shunt/files/shunt.uc

@@ -14,6 +14,8 @@ import { openlog, syslog, LOG_PID, LOG_DAEMON, LOG_ERR, LOG_WARNING,
 import { load as cfg_load, parse as cfg_parse, MIN as cfg_min } from 'shunt.config';
 import { compile as match_compile } from 'shunt.match';
 import { load as files_load, DIR as FILES_DIR } from 'shunt.list_file';
+import { PATH as LEARNED, fingerprints as learned_fps, save as learned_save,
+	load as learned_load } from 'shunt.learned';
 import { action_name, compile as nft_compile, flip as nft_flip, active as nft_active,
 	refresh, teardown,
 	addr_family, TABLE } from 'shunt.nft';
@@ -151,11 +153,51 @@ function nft_pipe(batch, what) {
 	return true;
 }
 
+// Seconds on CLOCK_MONOTONIC. Everything that measures a lifetime or a
+// duration uses it: the kernel's element timeouts do not follow the wall
+// clock either, and the wall clock jumps - at boot an OpenWrt box starts
+// from the newest file timestamp and NTP moves it, forward or back, while
+// shunt is already running. time() is left for what is shown as a date.
+function mono() {
+	return clock(true)[0];
+}
+
 // shunt's table check
 function table_present() {
 	return quiet([ 'nft', '-t', 'list', 'table', ...split(TABLE, ' ') ]) == 0;
 }
 
+// A write from snoop carries the TTL of the answer it was learned from; one
+// from poll carries none. Either way the element lives no longer than
+// entry_ttl and no shorter than its floor, so a zero TTL still gets a few
+// seconds of coverage and a week-long one does not pin a stale address.
+function queue_writes(st, writes, now) {
+	let max = st.state.cfg.global.entry_ttl;
+	let min = cfg_min.entry_ttl;
+	let fresh = 0;
+
+	for (let w in writes) {
+		if (!st.state.nft.learn[w.set])
+			continue;
+
+		// A kept element brings the time it has left, which may well be
+		// below the floor; it is not stretched to it.
+		let ttl = w.ttl ?? max;
+		w.ttl = (ttl < min && !w.kept) ? min : (ttl > max) ? max : ttl;
+
+		let d = st.cache.due(w.set, w.addr, now, w.ttl);
+
+		if (!d)
+			continue;
+
+		st.pending[`${w.set}/${w.addr}`] = w;
+		if (d == 'new')
+			fresh++;
+	}
+
+	return fresh;
+}
+
 // Re-creates the table if it went away
 function ensure_table(st) {
 	if (table_present()) {
@@ -173,8 +215,23 @@ function ensure_table(st) {
 
 	st.lost = false;
 	st.stats.reapplied++;
+
+	// The kernel lost the learned elements with the table, shunt did not:
+	// queue what is still running for the next write, with the time it has
+	// left. Not written from here - ensure_table() runs inside
+	// drain_writes() too.
+	let now = mono();
+	let keep = st.cache.dump(now);
+
 	st.cache.reset();
-	log('notice', 'ruleset re-applied - learned addresses refill from the next poll cycle and from snoop');
+	for (let e in keep) {
+		e.ttl = e.exp - now;
+		e.kept = true;
+	}
+	queue_writes(st, keep, now);
+
+	log('notice', sprintf('ruleset re-applied - %d learned address(es) queued to be written back',
+		length(keep)));
 
 	return 'recreated';
 }
@@ -191,26 +248,44 @@ function apply(state) {
 
 	rt_quiet(state.route.del);
 
+	// Devices of interfaces netifd knows but has not brought up. A route
+	// through one cannot be added before the device exists - at boot a
+	// tunnel usually does not yet - and its ifup rebuilds the routes
+	// anyway, so that failure is expected and not a warning. An interface
+	// netifd does not know at all has `up` null and is not excused.
+	let later = {};
+
+	for (let p in (state.resolved ?? []))
+		for (let mb in (p.members ?? []))
+			if (mb.up === false && mb.dev != null)
+				later[mb.dev] = true;
+
 	let rt = rtnl();
-	let failed = 0;
-	let reasons = {};
+	let reasons = {}, waiting = {};
 
 	for (let op in state.route.add) {
 		let why = rt ? rt_exec(rt, op) : 'ucode-mod-rtnl missing';
 
-		if (why != null) {
-			failed++;
+		if (why == null)
+			continue;
+
+		debug(sprintf('not applied: %s - %s', rt_describe(op), why));
+
+		let dev = op.msg?.oif;
+
+		if (rt && dev != null && later[dev])
+			waiting[dev] = (waiting[dev] ?? 0) + 1;
+		else
 			reasons[why] = (reasons[why] ?? 0) + 1;
-			debug(sprintf('not applied: %s - %s', rt_describe(op), why));
-		}
 	}
 
 	for (let why in reasons)
 		log('warn', sprintf('%d of %d route/rule operation(s) not applied - %s',
 			reasons[why], length(state.route.add), why));
 
-	if (failed)
-		log('warn', 'policy not applied yet - traffic falls through to main; restart once the interface is up');
+	if (length(waiting))
+		log('info', sprintf('routes through %s follow on ifup - not up yet',
+			join(', ', sort(keys(waiting)))));
 
 	return true;
 }
@@ -374,35 +449,19 @@ function build_state(silent) {
 // with large sets from another tool one add element costs seconds of CPU.
 // Writes are collected and applied by a timer whose interval follows the
 // measured cost: an ordinary box stays at the floor, an expensive one backs
-// off.
+// off. An address the sets have not seen yet does not wait for the timer
+// while the interval sits at the floor: its batch is written right after the
+// answer that brought it, at most once a second, so the further connections
+// a client opens after its lookup find the element in place. Refreshes of
+// known elements stay on the timer.
 const WRITE_MIN = 2;
 const WRITE_MAX = 60;
 const WRITE_FACTOR = 3;
 // Kernel timeouts run this much past the lifetime on top of the current
 // write interval: the replacement for an expiring element waits up to one
-// interval in the queue, and time() rounds to the second on both ends.
+// interval in the queue, and the clock rounds to the second on both ends.
 const WRITE_SLACK = 2;
 
-// A write from snoop carries the TTL of the answer it was learned from; one
-// from poll carries none. Either way the element lives no longer than
-// entry_ttl and no shorter than its floor, so a zero TTL still gets a few
-// seconds of coverage and a week-long one does not pin a stale address.
-function queue_writes(st, writes, now) {
-	let max = st.state.cfg.global.entry_ttl;
-	let min = cfg_min.entry_ttl;
-
-	for (let w in writes) {
-		if (!st.state.nft.learn[w.set])
-			continue;
-
-		let ttl = w.ttl ?? max;
-		w.ttl = (ttl < min) ? min : (ttl > max) ? max : ttl;
-
-		if (st.cache.due(w.set, w.addr, now, w.ttl))
-			st.pending[`${w.set}/${w.addr}`] = w;
-	}
-}
-
 function drain_writes(st) {
 	let due = values(st.pending);
 
@@ -418,9 +477,9 @@ function drain_writes(st) {
 	if (!length(r.batch))
 		return;
 
-	let t0 = time();
+	let t0 = mono();
 	let ok = nft_pipe(r.batch, 'refresh');
-	let cost = time() - t0;
+	let cost = mono() - t0;
 
 	if (ok)
 		debug(sprintf('%d element(s) written in %ds', length(due), cost));
@@ -443,11 +502,52 @@ function drain_writes(st) {
 	}
 }
 
+// Writes the queue now when it holds an address the sets have not seen. Only
+// at the floor interval - where it is above, a write costs seconds and the
+// timer is the throttle - and at most once per second, so a burst of new
+// answers on a busy segment is one nft run, not one per answer. A new
+// address that hits the limit goes out with one deferred run a second
+// later instead of waiting for the next tick.
+function drain_fresh(st, fresh) {
+	if (!fresh || st.interval > WRITE_MIN)
+		return;
+
+	let now = mono();
+
+	if (st.fresh_at != now) {
+		st.fresh_at = now;
+		drain_writes(st);
+		return;
+	}
+
+	if (st.fresh_wait)
+		return;
+
+	st.fresh_wait = true;
+	st.later(1000);
+}
+
 // The route renderer needs the probe tables the nft renderer numbered.
 function route_opts(state) {
 	return { ...state.opts, probes: state.nft.probes };
 }
 
+// The interface each policy's new connections take, null for the fallback.
+// nft_active() answers which member's mark is set, and with fallback
+// 'block' that is the first member even when all are dead - its table
+// then holds only the prohibit route.
+function in_use(chains, dead) {
+	let act = nft_active(chains, dead), out = {};
+
+	for (let pn, j in act) {
+		let mb = (j != null) ? chains[pn]?.members?.[j] : null;
+
+		out[pn] = (mb && !dead?.[mb.name]) ? mb.name : null;
+	}
+
+	return out;
+}
+
 // Link or probe state changed. Interfaces that became dead or alive are
 // handled as one batch: the mark chains of the affected policies are refilled
 // for the interface now in use, and the default routes of the changed
@@ -465,7 +565,7 @@ function apply_dead(state, wq, routes) {
 	if (!length(changed))
 		return;
 
-	let before = nft_active(state.nft.chains, state.opts.dead);
+	let before = in_use(state.nft.chains, state.opts.dead);
 
 	state.opts.dead = dead;
 
@@ -477,17 +577,16 @@ function apply_dead(state, wq, routes) {
 	if (length(batch) && !nft_pipe(batch, 'track'))
 		ensure_table(wq);
 
-	let after = nft_active(n.chains, dead);
+	let after = in_use(n.chains, dead);
 
 	for (let pn in sort(keys(after))) {
 		if (before[pn] === after[pn])
 			continue;
 
 		let p = filter(state.resolved, (x) => x.name == pn)[0];
-		let mb = (after[pn] != null) ? p?.members?.[after[pn]] : null;
 
-		log('notice', mb
-			? sprintf('policy %s: new connections use %s', pn, mb.name)
+		log('notice', (after[pn] != null)
+			? sprintf('policy %s: new connections use %s', pn, after[pn])
 			: sprintf('policy %s: no interface left - fallback %s', pn, p?.fallback ?? 'main'));
 	}
 
@@ -569,7 +668,26 @@ function run() {
 	let unresolved = {};
 
 	let wq = { state, cache, stats, pending: {}, lost: false,
-		interval: WRITE_MIN, timer: null };
+		interval: WRITE_MIN, timer: null, fresh_at: 0, fresh_wait: false,
+		later: null };
+
+	// The deferred run of drain_fresh(). The timer is kept and re-armed, not
+	// created per run, so nothing depends on how a given ucode release keeps
+	// an unreferenced timer alive.
+	let fresh_timer = null;
+
+	function fresh_run() {
+		wq.fresh_wait = false;
+		wq.fresh_at = mono();
+		drain_writes(wq);
+	}
+
+	wq.later = (ms) => {
+		if (fresh_timer)
+			fresh_timer.set(ms);
+		else
+			fresh_timer = uloop.timer(ms, fresh_run);
+	};
 
 	// Re-reads the list files, swaps the matcher and refills the address
 	// file sets - the two things a changed list needs. Nothing else moves:
@@ -637,7 +755,8 @@ function run() {
 			}
 		}
 
-		queue_writes(wq, poll_plan(res, state.matcher, targets), time());
+		drain_fresh(wq, queue_writes(wq, poll_plan(res, state.matcher,
+			targets), mono()));
 	}
 
 	// The cache is pruned here and not at the tail of poll_cycle():
@@ -650,11 +769,23 @@ function run() {
 		ensure_table(wq);
 
 		poll_cycle();
-		cache.prune(time());
+		cache.prune(mono());
 	}
 
 	wq.timer = uloop.interval(WRITE_MIN * 1000, () => drain_writes(wq));
 
+	let kept = learned_load(LEARNED,
+		learned_fps(state.cfg.policies, state.nft.learn), mono());
+
+	if (kept?.bad)
+		log('warn', sprintf('%s unreadable - learned addresses start empty', LEARNED));
+	else if (kept) {
+		queue_writes(wq, kept.writes, mono());
+		drain_writes(wq);
+		log('info', sprintf('%d learned address(es) carried over, %d dropped with a changed or removed policy',
+			length(kept.writes), kept.dropped));
+	}
+
 	poll_cycle();
 	uloop.interval(state.cfg.global.poll_interval * 1000, tick);
 
@@ -808,7 +939,7 @@ function run() {
 			// Each handler closes over its own socket; binding the loop
 			// variable would leave them all reading the last one opened.
 			uloop.handle(sock, () => {
-				let frame;
+				let frame, fresh = 0;
 
 				while ((frame = sock.recv(RECV_LEN, socket.MSG_DONTWAIT)) != null) {
 					let v = observe(frame, state.matcher);
@@ -830,8 +961,10 @@ function run() {
 
 					debug(sprintf('snoop: %s -> %s (%d addr)',
 						v.qname, join(', ', v.policies), length(writes)));
-					queue_writes(wq, writes, time());
+					fresh += queue_writes(wq, writes, mono());
 				}
+
+				drain_fresh(wq, fresh);
 			}, uloop.ULOOP_READ);
 		}
 
@@ -870,7 +1003,12 @@ function run() {
 			delete h.socks[fam];
 		}
 
+		// Every state change of a round goes into one apply_dead(): two
+		// tunnels failing in the same round are one flip, not a switch to
+		// the second one followed by the fallback.
 		function round() {
+			let changed = false;
+
 			for (let h in tracked) {
 				if (h.sent) {
 					let change = track_step(h, h.got);
@@ -881,7 +1019,7 @@ function run() {
 							? sprintf('track: %s does not answer - its policies fall back until it does', h.name)
 							: sprintf('track: %s answers again - its policies are back on it', h.name));
 						state.alive[h.name] = h.alive;
-						apply_dead(state, wq, true);
+						changed = true;
 					}
 				}
 
@@ -938,6 +1076,9 @@ function run() {
 					}
 				}
 			}
+
+			if (changed)
+				apply_dead(state, wq, true);
 		}
 
 		round();
@@ -957,6 +1098,9 @@ function run() {
 	uloop.run();
 
 	log('notice', 'stopping');
+	if (!learned_save(LEARNED, cache.dump(mono()),
+			learned_fps(state.cfg.policies, state.nft.learn)))
+		log('warn', sprintf('cannot write %s: %s', LEARNED, fs_error()));
 	for (let sock in socks)
 		sock.close();
 	for (let h in tracked)

shunt/src/dedupe.uc

@@ -13,7 +13,11 @@ export function create(entry_ttl) {
 	// lifetime it carries, in either direction: an answer with a long TTL
 	// extends an element that has aged past half, a short one cuts an
 	// element that poll or an earlier answer left long. Anything closer is
-	// a rewrite the kernel would not notice.
+	// a rewrite the kernel would not notice. A due write reports whether
+	// the pair is unknown or already past its expiry ('new' - prune runs
+	// only once per poll cycle, so a lapsed pair can still be on record) or
+	// only moves an expiry that is still running ('move'); the caller
+	// writes the first kind without waiting.
 	function due(set, addr, now, ttl) {
 		let k = `${set}/${addr}`;
 		let t = ttl ?? entry_ttl;
@@ -25,7 +29,7 @@ export function create(entry_ttl) {
 			return false;
 
 		exp[k] = e;
-		return true;
+		return (cur == null || cur <= now) ? 'new' : 'move';
 	}
 
 	function prune(now) {
@@ -47,9 +51,26 @@ export function create(entry_ttl) {
 		exp = {};
 	}
 
+	// Every pair still running, with its absolute expiry - what the kernel
+	// holds, as far as shunt wrote it. The key is `set/addr`; neither part
+	// can contain a slash.
+	function dump(now) {
+		let out = [];
+
+		for (let k, e in exp) {
+			if (e <= now)
+				continue;
+
+			let i = index(k, '/');
+			push(out, { set: substr(k, 0, i), addr: substr(k, i + 1), exp: e });
+		}
+
+		return out;
+	}
+
 	function size() {
 		return length(keys(exp));
 	}
 
-	return { due, prune, reset, size };
+	return { due, prune, reset, dump, size };
 };

shunt/src/learned.uc

@@ -0,0 +1,109 @@
+// shunt - learned addresses across a restart
+//
+// The learned sets live in shunt's own table, and every start destroys and
+// re-creates that table, so a reload would forget everything snoop picked
+// up and route those destinations over the normal uplink until a client asks
+// again. On the way out the daemon writes what it still holds to tmpfs, the
+// next start writes it back.
+//
+// An entry comes back only into the same set, with the time it has left,
+// and only if the policy's domain selectors are unchanged: a policy that now
+// names other domains must not inherit the addresses of the old ones. The
+// selectors are compared as configured - `domain` and the `domain_file`
+// names, not the file contents - which matches `shunt refresh`, where a
+// changed file keeps the learned addresses too.
+//
+// SPDX-License-Identifier: GPL-3.0-or-later
+// Copyright (c) 2026 Dirk Brenken <dev@brenken.org>
+
+import { readfile, writefile, rename, unlink } from 'fs';
+import { set_name } from 'shunt.nft';
+
+export const PATH = '/var/run/shunt.learned';
+
+// 2: expiries on CLOCK_MONOTONIC instead of the wall clock. A file of
+// another format is dropped without a word: it can only be left over from
+// the previous package version, one restart ago.
+const FORMAT = 2;
+
+// Selector fingerprint per learned set name.
+export function fingerprints(policies, learn) {
+	let out = {};
+
+	for (let p in (policies ?? [])) {
+		let fp = sprintf('%J', [ sort([ ...(p?.domains ?? []) ]),
+			sort([ ...(p?.domain_files ?? []) ]) ]);
+
+		for (let fam in [ 4, 6 ]) {
+			let s = set_name('d', fam, p?.name);
+
+			if (learn?.[s])
+				out[s] = fp;
+		}
+	}
+
+	return out;
+};
+
+// entries: [ { set, addr, exp } ], exp in CLOCK_MONOTONIC seconds - valid
+// within one boot, and the file in tmpfs does not outlive it either.
+export function save(path, entries, fps) {
+	let tmp = `${path}.tmp`;
+	let data = sprintf('%J', { format: FORMAT, fps, entries });
+
+	if (!writefile(tmp, data))
+		return false;
+	if (!rename(tmp, path)) {
+		unlink(tmp);
+		return false;
+	}
+
+	return true;
+};
+
+// Reads and removes the file - it is good for exactly one start. Returns
+// the writes to queue, each with the lifetime it has left, and the number
+// of entries dropped because their policy is gone or changed.
+export function load(path, fps, now) {
+	let raw = readfile(path);
+
+	if (raw == null)
+		return null;
+
+	unlink(path);
+
+	let d;
+
+	try {
+		d = json(raw);
+	}
+	catch (e) {
+		return { writes: [], dropped: 0, bad: true };
+	}
+
+	if (type(d) != 'object' || type(d.entries) != 'array')
+		return { writes: [], dropped: 0, bad: true };
+
+	if (d.format != FORMAT)
+		return { writes: [], dropped: 0, bad: false };
+
+	let writes = [], dropped = 0;
+
+	for (let e in d.entries) {
+		let left = (type(e?.exp) == 'int') ? e.exp - now : 0;
+
+		if (left <= 0)
+			continue;
+
+		let fp = fps[e.set];
+
+		if (fp == null || fp != d.fps?.[e.set]) {
+			dropped++;
+			continue;
+		}
+
+		push(writes, { set: e.set, addr: e.addr, ttl: left, kept: true });
+	}
+
+	return { writes, dropped, bad: false };
+};

shunt/src/nft.uc

@@ -159,6 +159,30 @@ function active_member(c, dead) {
 	return (c.fallback == 'main') ? null : 0;
 }
 
+// What a mark chain holds. With every interface dead and fallback 'block',
+// tcp is reset here, in the ruleset, before the first interface's mark sends
+// the rest to the prohibit route. The kernel rate-limits the ICMP errors
+// that route answers with, per client: one that keeps retrying flows into
+// the blocked policy uses up the budget, and its next connection attempt is
+// dropped silently and times out instead of failing - measured. A reset is
+// not rate-limited, so a tcp connection fails at once whatever else the
+// client sends. Only new flows get here, the settled ones return before.
+function chain_rules(c, dead) {
+	let j = active_member(c, dead);
+
+	if (j == null)
+		return [];
+
+	let out = [];
+
+	if (c.fallback == 'block' && dead?.[c.members[j].name])
+		push(out, 'meta l4proto tcp counter reject with tcp reset');
+
+	push(out, c.members[j].stmt);
+
+	return out;
+}
+
 function mask_shift(mask) {
 	let n = 0;
 	while (n < 32 && !((mask >> n) & 1))
@@ -460,7 +484,10 @@ export function compile(policies, opts) {
 		let chain = null;
 		let fb = p.fallback ?? 'main';
 
-		if (action == 'route' && (k > 1 ||
+		// A chain where the state of an interface changes what is marked:
+		// several interfaces, a tracked one with fallback 'main', and any
+		// with fallback 'block', whose chain resets tcp while it is dead.
+		if (action == 'route' && (k > 1 || fb == 'block' ||
 		    (probe_of[members[0]?.name] && fb == 'main'))) {
 			chain = `mk_${pname}`;
 			chains[pname] = { chain, fallback: fb, members: map(members,
@@ -718,11 +745,10 @@ export function compile(policies, opts) {
 
 	for (let n in sort(keys(chains))) {
 		let c = chains[n];
-		let j = active_member(c, opts?.dead);
 
 		push(mkchains, `\tchain ${c.chain} {`);
-		if (j != null)
-			push(mkchains, `\t\t${c.members[j].stmt}`);
+		for (let r in chain_rules(c, opts?.dead))
+			push(mkchains, `\t\t${r}`);
 		push(mkchains, '\t}');
 	}
 
@@ -758,11 +784,10 @@ export function flip(chains, dead) {
 
 	for (let n in sort(keys(chains ?? {}))) {
 		let c = chains[n];
-		let j = active_member(c, dead);
 
 		push(out, sprintf('flush chain %s %s', TABLE, c.chain));
-		if (j != null)
-			push(out, sprintf('add rule %s %s %s', TABLE, c.chain, c.members[j].stmt));
+		for (let r in chain_rules(c, dead))
+			push(out, sprintf('add rule %s %s %s', TABLE, c.chain, r));
 	}
 
 	return length(out) ? join('\n', out) + '\n' : '';

shunt/test.sh

@@ -8,6 +8,6 @@ ucode -e 'import * as a from "shunt.config"; import * as b from "shunt.nft";
 	import * as c from "shunt.dns"; import * as d from "shunt.frame";
 	import * as e from "shunt.match"; import * as f from "shunt.list_file";
 	import * as g from "shunt.route"; import * as h from "shunt.rt";
-	import * as i from "shunt.probe"; import * as j from "shunt.track";
+	import * as i from "shunt.probe"; import * as j from "shunt.track"; import * as k from "shunt.learned";
 	print("modules-ok\n");' |
 	grep 'modules-ok'