diff --git a/lua-periphery/Makefile b/lua-periphery/Makefile index c81fcd0ee..31a7ee26b 100644 --- a/lua-periphery/Makefile +++ b/lua-periphery/Makefile @@ -1,12 +1,12 @@ include $(TOPDIR)/rules.mk PKG_NAME:=lua-periphery -PKG_VERSION:=2.5.0 -PKG_RELEASE:=1 +PKG_VERSION:=2.5.1 +PKG_RELEASE:=2 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/vsergeev/lua-periphery.git -PKG_SOURCE_VERSION:=e91d515ebbf04bd4cfc10a3f4c7e1f12736350af +PKG_SOURCE_VERSION:=6c966f76a1ba280d71871d737f4802c34d72ff55 PKG_MIRROR_HASH:=skip PKG_LICENSE:=MIT diff --git a/luci-app-flowproxy/Makefile b/luci-app-flowproxy/Makefile index 70b7dcaf4..34d46ca70 100644 --- a/luci-app-flowproxy/Makefile +++ b/luci-app-flowproxy/Makefile @@ -1,19 +1,22 @@ include $(TOPDIR)/rules.mk PKG_NAME:=luci-app-flowproxy -PKG_VERSION:=1.0.4 -PKG_RELEASE:=4 +PKG_VERSION:=1.0.3 +PKG_RELEASE:=5 LUCI_TITLE:=LuCI Support for FlowProxy Traffic Diversion -LUCI_DEPENDS:=+luci-base +nftables +kmod-nft-core +ip-full +LUCI_DEPENDS:=+luci-base +nftables +ip-full LUCI_PKGARCH:=all +define Package/$(PKG_NAME)/conffiles +/etc/config/flowproxy +endef + define Package/$(PKG_NAME)/postinst #!/bin/sh -if [ -z "$${IPKG_INSTROOT}" ]; then - [ -f /etc/uci-defaults/99_flowproxy ] && ( . /etc/uci-defaults/99_flowproxy ) && rm -f /etc/uci-defaults/99_flowproxy - (sleep 1; /etc/init.d/rpcd reload) >/dev/null 2>&1 & -fi +[ -n "$${IPKG_INSTROOT}" ] || { + /etc/init.d/rpcd restart +} exit 0 endef diff --git a/luci-app-flowproxy/htdocs/luci-static/resources/view/flowproxy/flowproxy.js b/luci-app-flowproxy/htdocs/luci-static/resources/view/flowproxy/flowproxy.js index 96df74ce6..eb1449868 100644 --- a/luci-app-flowproxy/htdocs/luci-static/resources/view/flowproxy/flowproxy.js +++ b/luci-app-flowproxy/htdocs/luci-static/resources/view/flowproxy/flowproxy.js @@ -40,18 +40,6 @@ var callRestartService = rpc.declare({ method: 'restart_service' }); -var callGetDatasetInfo = rpc.declare({ - object: 'luci.flowproxy', - method: 'get_dataset_info', - params: ['file_path'] -}); - -var callUpdateDataset = rpc.declare({ - object: 'luci.flowproxy', - method: 'update_dataset', - params: ['url', 'file_path'] -}); - return L.view.extend({ load: function() { return Promise.all([ @@ -108,12 +96,6 @@ return L.view.extend({ }, [ _('Restart') ])); } else { content.push(E('span', { 'style': 'color: red; font-weight: bold;' }, [ _('stopped') ])); - content.push(' '); - content.push(E('button', { - 'class': 'btn cbi-button cbi-button-apply', - 'style': 'margin-left: 10px;', - 'click': ui.createHandlerFn(this, 'handleRestart') - }, [ _('Restart') ])); } L.dom.content(statusEl, content); } @@ -207,54 +189,11 @@ return L.view.extend({ s.taboption('settings', form.Flag, 'dns_proxy_enabled', _('Force upstream DNS to proxy server')).rmempty = false; o = s.taboption('settings', form.Value, 'proxy_server_ip_addr', _('Proxy server IP address')); + o.datatype = 'ip4addr'; o.rmempty = false; o.placeholder = '192.168.1.254'; - o.rmempty = true; o.validate = function(section_id, value) { - var isEnabled = (s.formvalue(section_id, 'enabled') === '1'); - - // 1. 如果值为空：仅当启用了服务尝试保存时才提示必填；未启用时允许留空，绝不展示刺眼红框 - if (!value || value.trim() === '') { - if (isEnabled) { - return _('Proxy server IP address is required when FlowProxy is enabled.'); - } - return true; - } - - value = value.trim(); - - // 2. 基础 IPv4 格式校验 - if (!value.match(/^([0-9]{1,3}\.){3}[0-9]{1,3}$/)) { - return _('Invalid IPv4 address format'); - } - var octets = value.split('.'); - for (var oi = 0; oi < 4; oi++) { - var oct = parseInt(octets[oi], 10); - if (isNaN(oct) || oct < 0 || oct > 255) { - return _('Invalid IPv4 address format'); - } - } - - if (value === '127.0.0.1' || value === '0.0.0.0') { - return _('Invalid proxy server IP address.'); - } - - // 3. 拦截本路由器自身的 IP 地址，彻底避免产生自指路由环路导致网络崩溃 - var allLocalIps = []; - for (var k = 0; k < ifaces.length; k++) { - var addrs = ifaces[k].getIPAddrs() || []; - for (var a = 0; a < addrs.length; a++) { - var pureIp = addrs[a].split('/')[0]; - if (pureIp && allLocalIps.indexOf(pureIp) === -1) { - allLocalIps.push(pureIp); - } - } - } - if (allLocalIps.indexOf(value) !== -1) { - return _('The proxy server IP address cannot be the local router IP (%s).').format(value); - } - var ipInCidr = function(ipStr, cidrStr) { - if (!cidrStr || cidrStr.indexOf(':') !== -1 || !ipStr || ipStr.indexOf(':') !== -1) return null; + if (!cidrStr || cidrStr.indexOf(':') !== -1 || !ipStr || ipStr.indexOf(':') !== -1) return false; var parts = cidrStr.split('/'); var cidrIp = parts[0]; var maskBits = parseInt(parts[1] || '32', 10); @@ -270,19 +209,15 @@ return L.view.extend({ var ipNum = ipToNum(ipStr); var cidrNum = ipToNum(cidrIp); - if (maskBits === 0) return { inSubnet: true, isBcastOrNet: false }; + + if (maskBits === 0) return true; var mask = (0xffffffff << (32 - maskBits)) >>> 0; - var inSubnet = ((ipNum & mask) === (cidrNum & mask)); - var netNum = (cidrNum & mask) >>> 0; - var bcastNum = (netNum | (~mask >>> 0)) >>> 0; - var isBcastOrNet = (maskBits < 31 && (ipNum === netNum || ipNum === bcastNum)); - - return { inSubnet: inSubnet, isBcastOrNet: isBcastOrNet }; + return (ipNum & mask) === (cidrNum & mask); }; var selected_interface = s.formvalue(section_id, 'interface'); - if (!selected_interface || selected_interface === 'auto') return true; + if (!value || !selected_interface) return true; var target_iface = null; for (var i = 0; i < ifaces.length; i++) { @@ -299,12 +234,9 @@ return L.view.extend({ if (!ips || ips.length === 0) return true; var inSubnet = false; - var isBcastOrNet = false; for (var j = 0; j < ips.length; j++) { - var res = ipInCidr(value, ips[j]); - if (res && res.inSubnet) { + if (ipInCidr(value, ips[j])) { inSubnet = true; - if (res.isBcastOrNet) isBcastOrNet = true; break; } } @@ -313,61 +245,19 @@ return L.view.extend({ return _('The proxy server IP address must be within the subnet of the selected interface (%s).').format(ips.join(', ')); } - if (isBcastOrNet) { - return _('The proxy server IP address cannot be the network address or broadcast address.'); - } - return true; }; o = s.taboption('settings', form.Value, 'proxy_server_dns_port', _('Proxy server DNS port')); o.datatype = 'port'; o.default = '5353'; o.rmempty = false; - var iface_opt = s.taboption('settings', form.ListValue, 'interface', _('Proxy server interface')); - iface_opt.value('auto', _('Auto (Detect automatically)')); - devices.forEach(function(d) { iface_opt.value(d.getName(), d.getName()); }); - iface_opt.default = 'auto'; - iface_opt.onchange = function(ev, sid, val) { - var ip_opt = m.lookupOption('proxy_server_ip_addr', sid)[0]; - if (ip_opt) { - var currVal = ip_opt.formvalue(sid); - // 仅当输入框已有非空输入时才触发重新比对子网，避免在空值时派发失焦导致误报红框 - if (currVal && currVal.trim() !== '') { - var el = document.getElementById(ip_opt.cbid(sid)); - if (el) { - var inputEl = el.tagName === 'INPUT' ? el : el.querySelector('input'); - if (inputEl) { - inputEl.dispatchEvent(new CustomEvent('input', { bubbles: true })); - } - } - } - } - }; + o = s.taboption('settings', form.ListValue, 'interface', _('Proxy server interface')); + devices.forEach(function(d) { o.value(d.getName(), d.getName()); }); + o.default = 'br-lan'; - o = s.taboption('settings', form.Value, 'traffic_mark', _('Traffic Mark')); - o.datatype = 'or(uinteger, hex)'; - o.default = '0x666'; - o.validate = function(section_id, value) { - if (!value) return true; - var num = parseInt(value, (typeof value === 'string' && value.indexOf('0x') === 0) ? 16 : 10); - if (isNaN(num) || num <= 0) return _('Traffic mark cannot be 0.'); - return true; - }; - - o = s.taboption('settings', form.Value, 'routing_table', _('Routing Table ID')); - o.datatype = 'and(uinteger,range(1, 4294967295))'; - o.default = '888'; - o.validate = function(section_id, value) { - var num = parseInt(value, 10); - if (num === 0 || num === 253 || num === 254 || num === 255) { - return _('Routing table ID %d is reserved by the system (0, 253, 254, 255). Please use another ID.').format(num); - } - return true; - }; - - o = s.taboption('settings', form.Value, 'rule_priority', _('Rule Priority')); - o.datatype = 'and(uinteger,range(10, 32765))'; - o.default = '1000'; + s.taboption('settings', form.Value, 'traffic_mark', _('Traffic Mark')).datatype = 'or(uinteger, hex)'; + s.taboption('settings', form.Value, 'routing_table', _('Routing Table ID')).datatype = 'and(uinteger,range(1, 4294967295))'; + s.taboption('settings', form.Value, 'rule_priority', _('Rule Priority')).datatype = 'and(uinteger,range(0, 4294967295))'; // --- Rules --- var nftsets = uci.sections('flowproxy', 'nftset').map(function(ss) { return '@' + ss['.name']; }); @@ -529,14 +419,16 @@ return L.view.extend({ } return true; }; - // 联动：当 match_type 改变时，仅在已有输入内容时重新校验 match_value + // 联动：当 match_type 改变时，强制触发 match_value 的校验 match_type.onchange = function(ev, sid, val) { var value_input = m.lookupOption('match_value', sid)[0]; if (value_input) setTimeout(function() { var el = document.getElementById(value_input.cbid(sid)); if (el) { var inputEl = el.tagName === 'INPUT' ? el : el.querySelector('input'); - if (inputEl && inputEl.value && inputEl.value.trim() !== '') { + if (inputEl) { + inputEl.dispatchEvent(new CustomEvent('change', { bubbles: true })); + inputEl.dispatchEvent(new CustomEvent('blur', { bubbles: true })); inputEl.dispatchEvent(new CustomEvent('input', { bubbles: true })); } } @@ -568,15 +460,7 @@ return L.view.extend({ var path = uci.get('flowproxy', sid, 'file_path') || '/usr/share/flowproxy/chnroute.txt'; var resNode = E('span', { 'style': 'margin-left: 10px; font-weight: bold; color: #444;', 'id': 'line-count-status' }, [ '...' ]); var input = node.querySelector('input'); if (input) input.parentNode.appendChild(resNode); - callGetDatasetInfo(path).then(function(res) { - if (res && res.exists) { - L.dom.content(resNode, [ _('(%d lines)').format(res.lines) ]); - } else { - L.dom.content(resNode, [ _('(file not found)') ]); - } - }).catch(function() { - L.dom.content(resNode, [ _('(n/a)') ]); - }); + fs.exec('/usr/bin/wc', ['-l', path]).then(function(res) { L.dom.content(resNode, (res.code === 0) ? [ _('(%d lines)').format(res.stdout.trim().split(' ')[0]) ] : [ _('(n/a)') ]); }); return node; }); }; @@ -590,21 +474,14 @@ return L.view.extend({ if (!path) path = '/usr/share/flowproxy/chnroute.txt'; if (!url) { ui.addNotification(null, E('p', _('Please set download_url first')), 'error'); return; } ui.showModal(null, [ E('p', { 'class': 'spinning', 'id': 'download-msg' }, [ _('Downloading chnroute data...') ]) ]); - return callUpdateDataset(url, path).then(function(res) { + return fs.exec('/usr/bin/wget', ['-q', '-O', path, url, '--timeout=10', '--no-check-certificate']).then(function(res) { var msgEl = document.getElementById('download-msg'); - if (res && res.success) { + if (res.code === 0) { if (msgEl) { msgEl.classList.remove('spinning'); L.dom.content(msgEl, [ _('Updated successfully.') ]); } - var cn = document.getElementById('line-count-status'); - if (cn) L.dom.content(cn, [ _('(%d lines)').format(res.lines || 0) ]); + var cn = document.getElementById('line-count-status'); if (cn) fs.exec('/usr/bin/wc', ['-l', path]).then(function(r) { L.dom.content(cn, (r.code === 0) ? [ _('(%d lines)').format(r.stdout.trim().split(' ')[0]) ] : [ _('(n/a)') ]); }); setTimeout(ui.hideModal, 1500); - } else { - ui.hideModal(); - ui.addNotification(null, E('p', _('Download failed: %s').format(res ? res.error : 'Unknown')), 'error'); - } - }).catch(function(e) { - ui.hideModal(); - ui.addNotification(null, E('p', _('Error: %s').format(e.message || String(e))), 'error'); - }); + } else { ui.hideModal(); ui.addNotification(null, E('p', _('Download failed')), 'error'); } + }).catch(function(e) { ui.hideModal(); ui.addNotification(null, E('p', _('Error: %s').format(e.message)), 'error'); }); }; var sg = s.taboption('lists', form.SectionValue, '_list_custom', form.GridSection, 'nftset', _('Custom nftables sets')); diff --git a/luci-app-flowproxy/po/templates/flowproxy.pot b/luci-app-flowproxy/po/templates/flowproxy.pot index 3c87ae8c5..0734ed227 100644 --- a/luci-app-flowproxy/po/templates/flowproxy.pot +++ b/luci-app-flowproxy/po/templates/flowproxy.pot @@ -118,7 +118,7 @@ msgstr "" msgid "Match Value" msgstr "" -msgid "Proxy server interface" +msgid "Network interface" msgstr "" msgid "Please set download_url first" @@ -145,9 +145,6 @@ msgstr "" msgid "Routing Table ID" msgstr "" -msgid "Rule Priority" -msgstr "" - msgid "Rules" msgstr "" @@ -229,49 +226,3 @@ msgstr "" msgid "this will delete ALL current %s rules and generate default templates. are you sure?" msgstr "" -msgid "(file not found)" -msgstr "" - -msgid "Download failed: %s" -msgstr "" - -msgid "The proxy server IP address must be within the subnet of the selected interface (%s)." -msgstr "" - -msgid "Invalid proxy server IP address." -msgstr "" - -msgid "The proxy server IP address cannot be the local router IP (%s)." -msgstr "" - -msgid "The proxy server IP address cannot be the network address or broadcast address." -msgstr "" - -msgid "Traffic mark cannot be 0." -msgstr "" - -msgid "Routing table ID %d is reserved by the system (0, 253, 254, 255). Please use another ID." -msgstr "" - -msgid "Restart" -msgstr "" - -msgid "Restarting FlowProxy service..." -msgstr "" - -msgid "FlowProxy service restarted successfully." -msgstr "" - -msgid "Failed to restart FlowProxy service." -msgstr "" - -msgid "RPC error: %s" -msgstr "" - -msgid "Proxy server IP address is required when FlowProxy is enabled." -msgstr "" - -msgid "Invalid IPv4 address format" -msgstr "" -msgid "Auto (Detect automatically)" -msgstr "" diff --git a/luci-app-flowproxy/po/zh_Hans/flowproxy.po b/luci-app-flowproxy/po/zh_Hans/flowproxy.po index 15f3c1fd8..1faa8dfff 100644 --- a/luci-app-flowproxy/po/zh_Hans/flowproxy.po +++ b/luci-app-flowproxy/po/zh_Hans/flowproxy.po @@ -250,31 +250,3 @@ msgstr "RPC 错误: %s" msgid "The proxy server IP address must be within the subnet of the selected interface (%s)." msgstr "代理服务器 IP 地址必须在所选网络接口的子网网段之内 (%s)。" -msgid "(file not found)" -msgstr "(文件不存在)" - -msgid "Download failed: %s" -msgstr "下载失败: %s" - -msgid "Invalid proxy server IP address." -msgstr "无效的代理服务器 IP 地址。" - -msgid "The proxy server IP address cannot be the local router IP (%s)." -msgstr "代理服务器 IP 地址不能为本路由器自身 IP (%s)。" - -msgid "The proxy server IP address cannot be the network address or broadcast address." -msgstr "代理服务器 IP 地址不能为网络地址或广播地址。" - -msgid "Traffic mark cannot be 0." -msgstr "流量标记不能为 0。" - -msgid "Routing table ID %d is reserved by the system (0, 253, 254, 255). Please use another ID." -msgstr "路由表 ID %d 为系统保留路由表 (0, 253, 254, 255)，请使用其他 ID。" - -msgid "Proxy server IP address is required when FlowProxy is enabled." -msgstr "启用代理分流时，代理服务器 IP 地址为必填项。" - -msgid "Invalid IPv4 address format" -msgstr "无效的 IPv4 地址格式" -msgid "Auto (Detect automatically)" -msgstr "自动探测" diff --git a/luci-app-flowproxy/root/etc/config/flowproxy b/luci-app-flowproxy/root/etc/config/flowproxy index a2e27eadb..8b3c0c6d2 100644 --- a/luci-app-flowproxy/root/etc/config/flowproxy +++ b/luci-app-flowproxy/root/etc/config/flowproxy @@ -3,7 +3,7 @@ config flowproxy 'global' option tcp_enabled '1' option udp_enabled '1' option proxy_server_ip_addr '' - option interface 'auto' + option interface 'br-lan' option traffic_mark '0x666' option routing_table '888' option rule_priority '1000' diff --git a/luci-app-flowproxy/root/etc/hotplug.d/iface/99-flowproxy b/luci-app-flowproxy/root/etc/hotplug.d/iface/99-flowproxy deleted file mode 100755 index a79955308..000000000 --- a/luci-app-flowproxy/root/etc/hotplug.d/iface/99-flowproxy +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/sh -# Hotplug trigger for flowproxy to restore policy routing on interface reload - -[ "$ACTION" = "ifup" ] || exit 0 - -enabled=$(uci -q get flowproxy.global.enabled || echo 0) -[ "$enabled" = "1" ] || exit 0 - -proxy_ip=$(uci -q get flowproxy.global.proxy_server_ip_addr) -[ -n "$proxy_ip" ] || exit 0 - -target_iface="" -if [ -f "/var/run/flowproxy.mark" ]; then - target_iface=$(cut -d':' -f4 /var/run/flowproxy.mark 2>/dev/null) -fi -if [ -z "$target_iface" ]; then - target_iface=$(uci -q get flowproxy.global.interface) - if [ -z "$target_iface" ] || [ "$target_iface" = "auto" ]; then - target_iface=$(ip route get "$proxy_ip" 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}') - fi -fi -target_iface="${target_iface:-br-lan}" - -if [ "$DEVICE" = "$target_iface" ] || [ "$INTERFACE" = "$target_iface" ]; then - [ -x /etc/init.d/flowproxy ] && /etc/init.d/flowproxy add_route -fi diff --git a/luci-app-flowproxy/root/etc/init.d/flowproxy b/luci-app-flowproxy/root/etc/init.d/flowproxy index 5ca0b657f..d628633ed 100755 --- a/luci-app-flowproxy/root/etc/init.d/flowproxy +++ b/luci-app-flowproxy/root/etc/init.d/flowproxy @@ -4,8 +4,6 @@ START=99 STOP=10 USE_PROCD=1 -EXTRA_COMMANDS="add_route" -EXTRA_HELP=" add_route Add routing rules and policy" NAME="flowproxy" UCI_CONFIG="flowproxy" @@ -59,62 +57,9 @@ check_env() { check_config() { local proxy_server_ip_addr=$(uci -q get "$UCI_CONFIG.global.proxy_server_ip_addr") [ -z "$proxy_server_ip_addr" ] && { log_msg error "proxy_server_ip_addr not configured"; return 1; } - - if [ "$proxy_server_ip_addr" = "127.0.0.1" ] || [ "$proxy_server_ip_addr" = "0.0.0.0" ]; then - log_msg error "Invalid proxy_server_ip_addr: $proxy_server_ip_addr"; return 1; - fi - - # 纵深防御：严禁将代理服务器设为本路由器自身的任一 IP，防止策略路由自指死循环导致网络崩溃 - if ip -4 -o addr show 2>/dev/null | awk '{print $4}' | cut -d/ -f1 | grep -Fxq "$proxy_server_ip_addr"; then - log_msg error "proxy_server_ip_addr ($proxy_server_ip_addr) is a local IP of this router, aborting to prevent routing loop" - return 1 - fi - - # 检查路由表 ID 是否为系统保留表 (0, 253, 254, 255) - local routing_table=$(uci -q get "$UCI_CONFIG.global.routing_table" || echo "888") - if [ "$routing_table" = "0" ] || [ "$routing_table" = "253" ] || [ "$routing_table" = "254" ] || [ "$routing_table" = "255" ]; then - log_msg error "routing_table ($routing_table) is a reserved system table (0, 253, 254, 255), aborting" - return 1 - fi - return 0 } -probe_proxy_server() { - local proxy_ip="$1" - local proxy_dns_port="$2" - local interface="$3" - - log_msg info "Probing proxy server $proxy_ip on interface $interface..." - - # 1. 检查 ARP 缓存或快速唤醒以获取 MAC 地址 - local mac="" - if [ -f "/proc/net/arp" ]; then - mac=$(awk -v ip="$proxy_ip" '$1 == ip { print $4; exit }' /proc/net/arp) - fi - - if [ -n "$mac" ] && [ "$mac" != "00:00:00:00:00:00" ]; then - log_msg info "Proxy server is REACHABLE: IP=$proxy_ip, MAC=$mac (found in ARP cache)" - else - # 快速单发 ping 1 秒唤醒 ARP 学习 - ping -c 1 -W 1 "$proxy_ip" >/dev/null 2>&1 - if [ -f "/proc/net/arp" ]; then - mac=$(awk -v ip="$proxy_ip" '$1 == ip { print $4; exit }' /proc/net/arp) - fi - if [ -n "$mac" ] && [ "$mac" != "00:00:00:00:00:00" ]; then - log_msg info "Proxy server is REACHABLE: IP=$proxy_ip, MAC=$mac" - else - log_msg warn "Proxy server $proxy_ip is NOT responding to ARP yet (it may be booting or offline, traffic will still be routed via policy table)" - fi - fi - - # 2. 若启用了上游 DNS 代理，在日志中记录目标地址与端口 - local dns_proxy=$(uci -q get "$UCI_CONFIG.global.dns_proxy_enabled" || echo 0) - if [ "$dns_proxy" = "1" ]; then - log_msg info "Upstream DNS redirection target configured: $proxy_ip#$proxy_dns_port" - fi -} - apply_nft_rules() { log_msg info "Generating nftables configuration..." local conf_level=$(uci -q get "$UCI_CONFIG.global.log_level" || echo "info") @@ -126,14 +71,10 @@ apply_nft_rules() { FLOWPROXY_LOG_LEVEL="$conf_level" "$GENERATE_SCRIPT" >/dev/null 2> "$err_file" local ret=$? - # 将脚本的输出记录到日志：成功时按 debug 输出，失败时按 error 输出 + # 将脚本的输出记录到日志 if [ -s "$err_file" ]; then while read -r line; do - if [ $ret -eq 0 ]; then - log_msg debug "Script: $line" - else - log_msg error "Generator: $line" - fi + log_msg debug "Script: $line" done < "$err_file" rm -f "$err_file" fi @@ -142,16 +83,14 @@ apply_nft_rules() { log_msg info "Applying nftables rules from $NFT_CONF" # 确保内核支持 inet 表 nft delete table inet flowproxy 2>/dev/null - local nft_err - nft_err=$(nft -f "$NFT_CONF" 2>&1) - if [ $? -eq 0 ]; then + if nft -f "$NFT_CONF"; then return 0 else - log_msg error "nftables rules application failed: $nft_err" + log_msg error "nftables rules application failed (syntax error)" return 1 fi else - log_msg error "Failed to generate nftables config (exit code: $ret)" + log_msg error "Failed to generate nftables config" return 1 fi else @@ -160,66 +99,33 @@ apply_nft_rules() { fi } -get_proxy_interface() { - local proxy_ip="$1" - local cfg_iface=$(uci -q get "$UCI_CONFIG.global.interface") - if [ -n "$cfg_iface" ] && [ "$cfg_iface" != "auto" ]; then - echo "$cfg_iface" - return - fi - - local auto_dev="" - if [ -n "$proxy_ip" ]; then - auto_dev=$(ip route get "$proxy_ip" 2>/dev/null | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}') - fi - echo "${auto_dev:-br-lan}" -} - add_route() { - check_config || return 1 - local proxy_server_ip_addr=$(uci -q get "$UCI_CONFIG.global.proxy_server_ip_addr") - local interface=$(get_proxy_interface "$proxy_server_ip_addr") - local traffic_mark=$(uci -q get "$UCI_CONFIG.global.traffic_mark") - traffic_mark="${traffic_mark:-0x666}" - local routing_table=$(uci -q get "$UCI_CONFIG.global.routing_table") - routing_table="${routing_table:-888}" - local rule_priority=$(uci -q get "$UCI_CONFIG.global.rule_priority") - rule_priority="${rule_priority:-1000}" + local interface=$(uci -q get "$UCI_CONFIG.global.interface" || echo "br-lan") + local traffic_mark=$(uci -q get "$UCI_CONFIG.global.traffic_mark" || echo "0x666") + local routing_table=$(uci -q get "$UCI_CONFIG.global.routing_table" || echo "888") + local rule_priority=$(uci -q get "$UCI_CONFIG.global.rule_priority" || echo "1000") # 1. 先尝试用旧记录清理一次，防止由于配置变更导致的残留 delete_route "quiet" - log_msg info "Setting up routing table $routing_table (dev: $interface) and rule for mark $traffic_mark with priority $rule_priority..." + log_msg info "Setting up routing table $routing_table and rule for mark $traffic_mark with priority $rule_priority..." - # 2. 记录当前使用的标记值、路由表 ID、优先级和实际出口网卡 - echo "$traffic_mark:$routing_table:$rule_priority:$interface" > "/var/run/flowproxy.mark" + # 2. 记录当前使用的标记值、路由表 ID 和优先级 + echo "$traffic_mark:$routing_table:$rule_priority" > "/var/run/flowproxy.mark" # 3. 再次确保新标记和表的规则是干净的 while ip rule del fwmark "$traffic_mark" lookup "$routing_table" 2>/dev/null; do :; done ip route flush table "$routing_table" 2>/dev/null # 4. 添加路由和策略 - local route_err rule_err - route_err=$(ip route add default via "$proxy_server_ip_addr" dev "$interface" table "$routing_table" 2>&1) - if [ $? -ne 0 ]; then - log_msg error "Failed to add default route to table $routing_table via $proxy_server_ip_addr (dev: $interface): $route_err" - log_msg error "Proxy server IP $proxy_server_ip_addr is unreachable or has no valid route on interface $interface" - return 1 - fi - - rule_err=$(ip rule add fwmark "$traffic_mark" lookup "$routing_table" priority "$rule_priority" 2>&1) + local err + ip route add default via "$proxy_server_ip_addr" dev "$interface" table "$routing_table" 2>/dev/null + err=$(ip rule add fwmark "$traffic_mark" lookup "$routing_table" priority "$rule_priority" 2>&1) if [ $? -eq 0 ]; then log_msg info "Route added: via $proxy_server_ip_addr dev $interface table $routing_table for mark $traffic_mark with priority $rule_priority" else - log_msg error "Failed to add ip rule: $rule_err" - return 1 - fi - - # 5. 禁用目标网卡的 ICMP 重定向通告，防止客户端绕行或 TCP 连接挂起 - if [ -d "/proc/sys/net/ipv4/conf/$interface" ]; then - sysctl -w "net.ipv4.conf.$interface.send_redirects=0" >/dev/null 2>&1 - log_msg debug "Disabled send_redirects for $interface" + log_msg error "Failed to add ip rule: $err" fi } @@ -228,23 +134,18 @@ delete_route() { local quiet="$1" # 1. 获取两组清理目标：旧值(记录文件) 和 新值(当前UCI) - local old_mark="" old_table="" old_prio="" old_iface="" + local old_mark="" old_table="" old_prio="" if [ -f "/var/run/flowproxy.mark" ]; then local saved_val=$(cat "/var/run/flowproxy.mark") old_mark=$(echo "$saved_val" | cut -d':' -f1) old_table=$(echo "$saved_val" | cut -d':' -f2) old_prio=$(echo "$saved_val" | cut -d':' -f3) - old_iface=$(echo "$saved_val" | cut -d':' -f4) [ "$quiet" != "quiet" ] && rm -f "/var/run/flowproxy.mark" fi - local curr_mark=$(uci -q get "$UCI_CONFIG.global.traffic_mark") - curr_mark="${curr_mark:-0x666}" - local curr_table=$(uci -q get "$UCI_CONFIG.global.routing_table") - curr_table="${curr_table:-888}" - local curr_prio=$(uci -q get "$UCI_CONFIG.global.rule_priority") - curr_prio="${curr_prio:-1000}" - local curr_iface=$(get_proxy_interface "$(uci -q get "$UCI_CONFIG.global.proxy_server_ip_addr")") + local curr_mark=$(uci -q get "$UCI_CONFIG.global.traffic_mark" || echo "0x666") + local curr_table=$(uci -q get "$UCI_CONFIG.global.routing_table" || echo "888") + local curr_prio=$(uci -q get "$UCI_CONFIG.global.rule_priority" || echo "1000") [ "$quiet" != "quiet" ] && log_msg info "Cleaning up FlowProxy routes (Atomic Mode)..." @@ -252,10 +153,6 @@ delete_route() { safe_del_rule() { local m="$1" t="$2" p="$3" [ -z "$m" ] || [ -z "$t" ] && return - if [ "$t" = "0" ] || [ "$t" = "253" ] || [ "$t" = "254" ] || [ "$t" = "255" ]; then - log_msg error "Refusing to flush reserved system routing table $t" - return - fi log_msg debug "Attempting cleanup: mark=$m, table=$t, priority=${p:-any}" # 精准清理 (带优先级) [ -n "$p" ] && while ip rule del fwmark "$m" lookup "$t" priority "$p" 2>/dev/null; do @@ -274,12 +171,6 @@ delete_route() { safe_del_rule "$old_mark" "$old_table" "$old_prio" [ "$old_mark" != "$curr_mark" ] || [ "$old_table" != "$curr_table" ] && \ safe_del_rule "$curr_mark" "$curr_table" "$curr_prio" - - # 恢复 ICMP 重定向通告 - [ -n "$old_iface" ] && [ -d "/proc/sys/net/ipv4/conf/$old_iface" ] && \ - sysctl -w "net.ipv4.conf.$old_iface.send_redirects=1" >/dev/null 2>&1 - [ -n "$curr_iface" ] && [ "$curr_iface" != "$old_iface" ] && [ -d "/proc/sys/net/ipv4/conf/$curr_iface" ] && \ - sysctl -w "net.ipv4.conf.$curr_iface.send_redirects=1" >/dev/null 2>&1 } delete_nft() { @@ -301,46 +192,68 @@ setup_dns() { [ -z "$proxy_server_ip_addr" ] && { log_msg error "proxy_server_ip_addr not configured, cannot setup DNS"; return 1; } - log_msg info "Setting up DNS redirection to $proxy_server_ip_addr#$proxy_server_dns_port via drop-in config..." + log_msg info "Setting up DNS redirection to $proxy_server_ip_addr#$proxy_server_dns_port..." - mkdir -p /tmp/dnsmasq.d - cat > /tmp/dnsmasq.d/flowproxy.conf << EOF -# Generated by FlowProxy - DO NOT EDIT MANUALLY -no-resolv -server=$proxy_server_ip_addr#$proxy_server_dns_port -EOF - - # 同步分发至各个可能存在的 dnsmasq 实例目录 - for dir in /tmp/dnsmasq*.d; do - if [ -d "$dir" ] && [ "$dir" != "/tmp/dnsmasq.d" ]; then - cp -f /tmp/dnsmasq.d/flowproxy.conf "$dir/flowproxy.conf" - fi - done - - # 历史遗留自愈：若早期版本存在残留的 dns_backup 节，静默移除避免脏数据 - if uci -q get "$UCI_CONFIG.dns_backup" >/dev/null; then - uci -q delete "$UCI_CONFIG.dns_backup" - uci -q commit "$UCI_CONFIG" + # Persistent backup in UCI + if ! uci -q get "$UCI_CONFIG.dns_backup" >/dev/null; then + local old_noresolv=$(uci -q get dhcp.@dnsmasq[0].noresolv) + local old_dns_redirect=$(uci -q get dhcp.@dnsmasq[0].dns_redirect) + local old_servers=$(uci -q get dhcp.@dnsmasq[0].server | tr '\n' ',' | sed 's/,$//') + + log_msg debug "Backing up DNS: noresolv=${old_noresolv:-empty}, redirect=${old_dns_redirect:-empty}, servers=${old_servers:-empty}" + + uci set "$UCI_CONFIG.dns_backup=dns_backup" + [ -n "$old_noresolv" ] && uci set "$UCI_CONFIG.dns_backup.noresolv=$old_noresolv" + [ -n "$old_dns_redirect" ] && uci set "$UCI_CONFIG.dns_backup.dns_redirect=$old_dns_redirect" + [ -n "$old_servers" ] && uci set "$UCI_CONFIG.dns_backup.servers=$old_servers" + uci commit "$UCI_CONFIG" fi + log_msg debug "Applying new DNS: noresolv=1, redirect=0, server=$proxy_server_ip_addr#$proxy_server_dns_port" + uci set dhcp.@dnsmasq[0].noresolv='1' + uci set dhcp.@dnsmasq[0].dns_redirect='0' + + # Delete current servers and add our proxy + uci -q delete dhcp.@dnsmasq[0].server + uci add_list dhcp.@dnsmasq[0].server="$proxy_server_ip_addr#$proxy_server_dns_port" + + uci commit dhcp /etc/init.d/dnsmasq restart - log_msg info "DNS drop-in configuration applied successfully" } cleanup_dns() { - log_msg info "Cleaning up DNS redirection drop-in configuration..." - rm -f /tmp/dnsmasq.d/flowproxy.conf - for dir in /tmp/dnsmasq*.d; do - [ -d "$dir" ] && rm -f "$dir/flowproxy.conf" - done - - # 历史遗留自愈 if uci -q get "$UCI_CONFIG.dns_backup" >/dev/null; then - uci -q delete "$UCI_CONFIG.dns_backup" - uci -q commit "$UCI_CONFIG" - fi + log_msg info "Restoring original DNS configuration..." + + local noresolv=$(uci -q get "$UCI_CONFIG.dns_backup.noresolv") + local dns_redirect=$(uci -q get "$UCI_CONFIG.dns_backup.dns_redirect") + local servers=$(uci -q get "$UCI_CONFIG.dns_backup.servers") + + log_msg debug "Restoring DNS values: noresolv=${noresolv:-0}, redirect=${dns_redirect:-1}, servers=${servers:-empty}" - /etc/init.d/dnsmasq restart + # Restore noresolv (default to 0 if not backed up) + [ -n "$noresolv" ] && uci set dhcp.@dnsmasq[0].noresolv="$noresolv" || uci set dhcp.@dnsmasq[0].noresolv="0" + + # Restore dns_redirect (only restore if it originally existed, avoid polluting config) + if [ -n "$dns_redirect" ]; then + uci set dhcp.@dnsmasq[0].dns_redirect="$dns_redirect" + else + uci -q delete dhcp.@dnsmasq[0].dns_redirect + fi + + # Restore servers correctly splitting comma-separated list + uci -q delete dhcp.@dnsmasq[0].server + if [ -n "$servers" ]; then + for s in $(echo "$servers" | tr ',' ' '); do + [ -n "$s" ] && uci add_list dhcp.@dnsmasq[0].server="$s" + done + fi + + uci commit dhcp + uci delete "$UCI_CONFIG.dns_backup" + uci commit "$UCI_CONFIG" + /etc/init.d/dnsmasq restart + fi } start_service() { @@ -348,42 +261,23 @@ start_service() { if [ "$enabled" -eq 0 ]; then log_msg info "FlowProxy is disabled, ensuring service is stopped" - [ -x /etc/init.d/flowproxy ] && /etc/init.d/flowproxy disable 2>/dev/null stop_service return 0 fi - # 状态自愈：当 enabled=1 时，确保服务已在 rc.d 中 enable，防止 luci-reload 因未启用而静默跳过执行 - [ -x /etc/init.d/flowproxy ] && /etc/init.d/flowproxy enable 2>/dev/null - log_msg info "Starting FlowProxy..." check_env || return 1 check_config || return 1 - - local proxy_server_ip_addr=$(uci -q get "$UCI_CONFIG.global.proxy_server_ip_addr") - local proxy_server_dns_port=$(uci -q get "$UCI_CONFIG.global.proxy_server_dns_port" || echo 5353) - local interface=$(get_proxy_interface "$proxy_server_ip_addr") - probe_proxy_server "$proxy_server_ip_addr" "$proxy_server_dns_port" "$interface" - apply_nft_rules || return 1 - if ! add_route; then - log_msg error "Routing setup failed, aborting and cleaning up service..." - stop_service - return 1 - fi + add_route setup_dns touch "$STATE_FILE" log_msg info "FlowProxy service started successfully" } - stop_service() { log_msg info "Stopping FlowProxy..." - local enabled=$(uci -q get "$UCI_CONFIG.global.enabled" || echo 0) - if [ "$enabled" -eq 0 ]; then - [ -x /etc/init.d/flowproxy ] && /etc/init.d/flowproxy disable 2>/dev/null - fi # 强制先移除标记，防止状态误报 rm -f "$STATE_FILE" delete_route @@ -392,15 +286,13 @@ stop_service() { log_msg info "FlowProxy service stopped" } -reload_service() { - log_msg info "Reloading FlowProxy..." +restart_service() { stop_service start_service } -restart_service() { - stop_service - start_service +reload_service() { + restart_service } service_triggers() { @@ -408,11 +300,4 @@ service_triggers() { procd_add_reload_trigger "$UCI_CONFIG" # 监听防火墙重载事件，防止规则被 fw4 清空后无法自动恢复 procd_add_reload_trigger "firewall" - # 监听网络接口事件（procd trigger 需接收 UCI 逻辑网络名，如 lan/wan） - local dev=$(get_proxy_interface "$(uci -q get "$UCI_CONFIG.global.proxy_server_ip_addr")") - local net_name="" - if command -v ubus >/dev/null 2>&1 && [ -n "$dev" ]; then - net_name=$(ubus call network.interface dump 2>/dev/null | jsonfilter -e "@.interface[@.device=\"$dev\"].interface" -e "@.interface[@.l3_device=\"$dev\"].interface" 2>/dev/null | head -n 1) - fi - procd_add_reload_interface_trigger "${net_name:-lan}" } diff --git a/luci-app-flowproxy/root/etc/uci-defaults/99_flowproxy b/luci-app-flowproxy/root/etc/uci-defaults/99_flowproxy deleted file mode 100644 index 1dbb9f736..000000000 --- a/luci-app-flowproxy/root/etc/uci-defaults/99_flowproxy +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh -[ ! -f "/usr/share/ucitrack/luci-app-flowproxy.json" ] && { - cat > /usr/share/ucitrack/luci-app-flowproxy.json << EEOF -{ - "config": "flowproxy", - "init": "flowproxy" -} -EEOF -} - -uci -q batch <<-EOF >/dev/null - delete ucitrack.@flowproxy[-1] - add ucitrack flowproxy - set ucitrack.@flowproxy[-1].init=flowproxy - commit ucitrack -EOF - -/etc/init.d/flowproxy enabled || /etc/init.d/flowproxy enable 2>/dev/null - -rm -f /tmp/luci-indexcache* 2>/dev/null -rm -f /tmp/luci-modulecache/* 2>/dev/null - -exit 0 diff --git a/luci-app-flowproxy/root/usr/share/flowproxy/generate_nft.sh b/luci-app-flowproxy/root/usr/share/flowproxy/generate_nft.sh index 6103f7472..dcf0e1f18 100755 --- a/luci-app-flowproxy/root/usr/share/flowproxy/generate_nft.sh +++ b/luci-app-flowproxy/root/usr/share/flowproxy/generate_nft.sh @@ -44,15 +44,20 @@ gen_set_definition() { [ -n "$uci_elems" ] && { [ -n "$elems" ] && elems="${elems}, "; elems="${elems}${uci_elems}"; } file_path=$(uci -q get "$CONFIG.$section.file_path") - if [ -n "$file_path" ]; then - if [ -f "$file_path" ]; then - local file_elems=$(get_file_elements "$file_path") - if [ -n "$file_elems" ]; then - [ -n "$elems" ] && elems="${elems}, " - elems="${elems}${file_elems}" - fi - else - log_debug "Set file $file_path not found, continuing without file elements" + download_url=$(uci -q get "$CONFIG.$section.download_url") + if [ -n "$download_url" ] && [ -n "$file_path" ]; then + if [ ! -f "$file_path" ]; then + echo "Downloading $section data from $download_url to $file_path..." >&2 + wget -q -O "$file_path" "$download_url" --timeout=10 --no-check-certificate + [ $? -ne 0 ] && { echo "Failed to download $section data" >&2; rm -f "$file_path"; } + fi + fi + + if [ -n "$file_path" ] && [ -f "$file_path" ]; then + local file_elems=$(get_file_elements "$file_path") + if [ -n "$file_elems" ]; then + [ -n "$elems" ] && elems="${elems}, " + elems="${elems}${file_elems}" fi fi @@ -178,68 +183,15 @@ TCP_ENABLED=$(uci -q get "$CONFIG.global.tcp_enabled" || echo "1") UDP_ENABLED=$(uci -q get "$CONFIG.global.udp_enabled" || echo "1") TRAFFIC_MARK=$(uci -q get "$CONFIG.global.traffic_mark" || echo "0x666") PROXY_SERVER_IP_ADDR=$(uci -q get "$CONFIG.global.proxy_server_ip_addr") -get_lan_devices() { - local dev="" devs="" nets="" - local z_idx=0 - while :; do - local z_name=$(uci -q get firewall.@zone[$z_idx].name) - [ -z "$z_name" ] && break - if [ "$z_name" = "lan" ]; then - nets=$(uci -q get firewall.@zone[$z_idx].network) - break - fi - z_idx=$((z_idx + 1)) - done - - [ -z "$nets" ] && nets="lan" - - for net in $nets; do - dev="" - if command -v ubus >/dev/null 2>&1; then - dev=$(ubus call network.interface."$net" status 2>/dev/null | jsonfilter -e "@.l3_device" 2>/dev/null) - [ -z "$dev" ] && dev=$(ubus call network.interface."$net" status 2>/dev/null | jsonfilter -e "@.device" 2>/dev/null) - fi - [ -z "$dev" ] && dev=$(uci -q get network."$net".device || uci -q get network."$net".device) - if [ -n "$dev" ]; then - case " $devs " in - *" $dev "*) ;; - *) devs="${devs}${devs:+ }$dev" ;; - esac - fi - done - - echo "${devs:-br-lan}" -} - -render_nft_lan_filter() { - local devs="$1" - set -- $devs - local count=$# - if [ "$count" -le 1 ]; then - echo " iifname != \"$1\" return" - else - local set_str="" - for d in "$@"; do - [ -n "$set_str" ] && set_str="${set_str}, " - set_str="${set_str}\"${d}\"" - done - echo " iifname != { $set_str } return" - fi -} - -LAN_DEVICES=$(get_lan_devices) -LAN_NFT_FILTER=$(render_nft_lan_filter "$LAN_DEVICES") if [ "$TCP_ENABLED" = "1" ]; then cat >> "$OUTPUT_FILE" << EOF chain LAN_MARKFLOW_TCP { type filter hook prerouting priority mangle; policy accept; -$LAN_NFT_FILTER fib daddr type { unspec, local, anycast, multicast } counter return EOF if [ -n "$PROXY_SERVER_IP_ADDR" ]; then - echo " ip saddr $PROXY_SERVER_IP_ADDR counter return" >> "$OUTPUT_FILE" - echo " ip daddr $PROXY_SERVER_IP_ADDR counter return" >> "$OUTPUT_FILE" + echo " ip saddr $PROXY_SERVER_IP_ADDR ip protocol tcp counter return" >> "$OUTPUT_FILE" fi for s in $SECTIONS_TCP; do process_rule "$s" "tcp" >> "$OUTPUT_FILE"; done echo " ip protocol tcp counter meta mark set $TRAFFIC_MARK" >> "$OUTPUT_FILE" @@ -250,12 +202,10 @@ if [ "$UDP_ENABLED" = "1" ]; then cat >> "$OUTPUT_FILE" << EOF chain LAN_MARKFLOW_UDP { type filter hook prerouting priority mangle; policy accept; -$LAN_NFT_FILTER fib daddr type { unspec, local, anycast, multicast } counter return EOF if [ -n "$PROXY_SERVER_IP_ADDR" ]; then - echo " ip saddr $PROXY_SERVER_IP_ADDR counter return" >> "$OUTPUT_FILE" - echo " ip daddr $PROXY_SERVER_IP_ADDR counter return" >> "$OUTPUT_FILE" + echo " ip saddr $PROXY_SERVER_IP_ADDR ip protocol udp counter return" >> "$OUTPUT_FILE" fi for s in $SECTIONS_UDP; do process_rule "$s" "udp" >> "$OUTPUT_FILE"; done echo " ip protocol udp counter meta mark set $TRAFFIC_MARK" >> "$OUTPUT_FILE" diff --git a/luci-app-flowproxy/root/usr/share/rpcd/acl.d/luci-app-flowproxy.json b/luci-app-flowproxy/root/usr/share/rpcd/acl.d/luci-app-flowproxy.json index dac28927e..c9d691f19 100644 --- a/luci-app-flowproxy/root/usr/share/rpcd/acl.d/luci-app-flowproxy.json +++ b/luci-app-flowproxy/root/usr/share/rpcd/acl.d/luci-app-flowproxy.json @@ -13,6 +13,11 @@ }, "write": { "uci": [ "flowproxy" ], + "file": { + "/usr/bin/wget": [ "exec" ], + "/usr/bin/wc": [ "exec" ], + "/usr/share/flowproxy/generate_nft.sh": [ "exec" ] + }, "ubus": { "luci.flowproxy": [ "*" ] } diff --git a/luci-app-flowproxy/root/usr/share/rpcd/ucode/luci.flowproxy b/luci-app-flowproxy/root/usr/share/rpcd/ucode/luci.flowproxy index 0a7e3b7ce..45f1aad6d 100755 --- a/luci-app-flowproxy/root/usr/share/rpcd/ucode/luci.flowproxy +++ b/luci-app-flowproxy/root/usr/share/rpcd/ucode/luci.flowproxy @@ -9,7 +9,6 @@ const STATE_FILE = '/var/run/flowproxy.running'; const IP_CACHE = '/tmp/flowproxy/lan_ip.cache'; const LOG_FILE = '/var/log/flowproxy.log'; const NFT_TABLE = 'flowproxy'; -const DNS_DROPIN_FILE = '/tmp/dnsmasq.d/flowproxy.conf'; // 辅助函数：安全 trim function safe_trim(str) { @@ -19,10 +18,10 @@ function safe_trim(str) { const methods = { get_status: { call: function() { - // 1. 快速探测标记文件 + // 1. 快速探测 let is_running = access(STATE_FILE, 'f'); - // 2. 自愈：检查内核状态 (nft 链与策略路由) + // 2. 自愈：检查内核状态并同步标记文件 let tcp_active = (system('nft list chain inet ' + NFT_TABLE + ' LAN_MARKFLOW_TCP >/dev/null 2>&1') === 0); let udp_active = (system('nft list chain inet ' + NFT_TABLE + ' LAN_MARKFLOW_UDP >/dev/null 2>&1') === 0); let real_running = (tcp_active || udp_active); @@ -41,6 +40,7 @@ const methods = { if (ubus_conn) { let ifaces = ubus_conn.call('network.interface', 'dump'); if (type(ifaces?.interface) === 'array') { + // 优先寻找名为 'lan' 的接口，或者寻找拥有 ipv4 地址的第一个非回环接口 let fallback_ip = ''; for (let iface in ifaces.interface) { if (iface.interface === 'lan' && iface['ipv4-address']?.[0]?.address) { @@ -64,8 +64,9 @@ const methods = { } let uci_cursor = cursor(); + let proxy_server_ip_addr = uci_cursor.get(UCI_CONFIG, 'global', 'proxy_server_ip_addr') || ''; - let dns_running = access(DNS_DROPIN_FILE, 'f') ? 1 : 0; + let dns_running = (uci_cursor.get(UCI_CONFIG, 'dns_backup') ? 1 : 0); return { running: real_running ? 1 : 0, @@ -113,67 +114,6 @@ const methods = { } }, - get_dataset_info: { - args: { file_path: 'string' }, - call: function(request) { - let file_path = request.args?.file_path || '/usr/share/flowproxy/chnroute.txt'; - if (!match(file_path, /^\/(usr\/share\/flowproxy|tmp\/flowproxy)\/[a-zA-Z0-9_.-]+$/)) { - return { error: 'Invalid path' }; - } - if (!access(file_path, 'r')) { - return { exists: false, lines: 0, size: 0 }; - } - let s = stat(file_path); - let lines = 0; - let fd = popen(sprintf("wc -l '%s' 2>/dev/null", file_path)); - if (fd) { - let out = fd.read('all'); - fd.close(); - if (out) { - let parts = split(trim(out), /\s+/); - lines = int(parts[0]) || 0; - } - } - return { exists: true, lines: lines, size: s ? s.size : 0 }; - } - }, - - update_dataset: { - args: { url: 'string', file_path: 'string' }, - call: function(request) { - let url = request.args?.url; - let file_path = request.args?.file_path || '/usr/share/flowproxy/chnroute.txt'; - if (!url || !match(url, /^https?:\/\/[A-Za-z0-9_.\/:~%#?&=+-]+$/)) { - return { success: false, error: 'Invalid URL' }; - } - if (!match(file_path, /^\/(usr\/share\/flowproxy|tmp\/flowproxy)\/[a-zA-Z0-9_.-]+$/)) { - return { success: false, error: 'Invalid target file path' }; - } - let tmp_file = file_path + '.tmp'; - let cmd = sprintf("wget -q -O '%s' '%s' --timeout=15 --no-check-certificate 2>&1", tmp_file, url); - let rc = system(cmd); - if (rc === 0 && access(tmp_file, 'f')) { - let s = stat(tmp_file); - if (s && s.size > 0) { - system(sprintf("mv -f '%s' '%s'", tmp_file, file_path)); - let lines = 0; - let fd = popen(sprintf("wc -l '%s' 2>/dev/null", file_path)); - if (fd) { - let out = fd.read('all'); - fd.close(); - if (out) { - let parts = split(trim(out), /\s+/); - lines = int(parts[0]) || 0; - } - } - return { success: true, lines: lines }; - } - } - system(sprintf("rm -f '%s'", tmp_file)); - return { success: false, error: 'Download failed or file empty' }; - } - }, - generate_nft_config: { call: function() { system('mkdir -p /tmp/flowproxy'); @@ -185,18 +125,19 @@ const methods = { get_runtime_config: { call: function() { let res_file = '/tmp/flowproxy/runtime.tmp'; + let dns_file = '/tmp/flowproxy/dns_runtime.tmp'; + system('mkdir -p /tmp/flowproxy'); system('/usr/share/flowproxy/generate_nft.sh runtime > ' + res_file + ' 2>&1'); let nft_runtime = readfile(res_file) || "(failed)"; system('rm -f ' + res_file); - let dns_runtime = "\n--- [ DNSmasq Drop-in Runtime Configuration ] ---\n"; - let dns_content = readfile(DNS_DROPIN_FILE); - if (dns_content) { - dns_runtime += dns_content; - } else { - dns_runtime += "(no active dnsmasq drop-in overrides)"; - } + system('uci show dhcp.@dnsmasq[0] | grep -E "noresolv|dns_redirect|server" | sed -E "s/^dhcp\\.[^.]*\\.([^=]*)=(.*)/\\1: \\2/; s/\\x27//g; s/#/:/g" > ' + dns_file + ' 2>&1'); + let dns_out = readfile(dns_file); + system('rm -f ' + dns_file); + + let dns_runtime = "\n--- [ DNSmasq Runtime Configuration ] ---\n"; + dns_runtime += (dns_out || "(no active dnsmasq overrides)"); return { runtime: safe_trim(nft_runtime + dns_runtime) }; } @@ -208,6 +149,7 @@ const methods = { try { let ubus_conn = connect(); if (ubus_conn) { + // 使用 network.device status 以获取所有物理设备 (包括 eth0 等 bridge 成员) let dev_status = ubus_conn.call('network.device', 'status'); if (dev_status) { for (let name in dev_status) { diff --git a/luci-app-ipsec-server/Makefile b/luci-app-ipsec-server/Makefile index c2db24c45..beacbf141 100644 --- a/luci-app-ipsec-server/Makefile +++ b/luci-app-ipsec-server/Makefile @@ -6,13 +6,13 @@ include $(TOPDIR)/rules.mk PKG_NAME:=luci-app-ipsec-server -PKG_VERSION:=20211223 -PKG_RELEASE:=1 +PKG_VERSION:=20260815 +PKG_RELEASE:=2 PKG_MAINTAINER:=Lienol LUCI_TITLE:=LuCI support for IPSec VPN Server -LUCI_DEPENDS:=+kmod-tun +luci-lib-jsonc +strongswan +strongswan-minimal +strongswan-mod-kernel-libipsec +strongswan-mod-openssl +strongswan-mod-xauth-generic +xl2tpd +LUCI_DEPENDS:=+kmod-tun +luci-lib-jsonc +strongswan +strongswan-minimal +strongswan-mod-kernel-libipsec +strongswan-mod-openssl +strongswan-mod-xauth-generic +strongswan-mod-eap-mschapv2 +strongswan-mod-eap-identity +strongswan-mod-pem +strongswan-mod-pkcs1 +strongswan-mod-pkcs8 +strongswan-mod-dhcp +strongswan-mod-farp LUCI_PKGARCH:=all define Package/$(PKG_NAME)/conffiles diff --git a/luci-app-ipsec-server/example.config b/luci-app-ipsec-server/example.config new file mode 100644 index 000000000..8525d32ad --- /dev/null +++ b/luci-app-ipsec-server/example.config @@ -0,0 +1,26 @@ +conn ikev2-EAP + keyexchange=ikev2 + auto=add + fragmentation=yes + forceencaps=yes + + left=%defaultroute + leftid=@ns.miaogongzi.cc + leftauth=pubkey + leftcert=full.pem + leftsendcert=always + leftsubnet=0.0.0.0/0 + + right=%any + rightid=%any + rightauth=eap-mschapv2 + eap_identity=%identity + rightsourceip=10.0.20.0/24 + rightdns=10.0.0.1 + + ike=aes256-sha256-modp1024,aes256-sha1-modp1024,aes128-sha256-modp1024,aes128-sha1-modp1024,aes256-sha256-modp2048,aes256-sha1-modp2048 + esp=aes256-sha256,aes128-sha256,aes256-sha1,aes128-sha1 + dpdaction=clear + dpddelay=30s + rekey=no + compress=no \ No newline at end of file diff --git a/luci-app-ipsec-server/luasrc/controller/ipsec-server.lua b/luci-app-ipsec-server/luasrc/controller/ipsec-server.lua index e9a271af4..67906fdda 100644 --- a/luci-app-ipsec-server/luasrc/controller/ipsec-server.lua +++ b/luci-app-ipsec-server/luasrc/controller/ipsec-server.lua @@ -11,14 +11,12 @@ function index() entry({"admin", "vpn", "ipsec-server", "settings"}, cbi("ipsec-server/settings"), _("General Settings"), 10).leaf = true entry({"admin", "vpn", "ipsec-server", "users"}, cbi("ipsec-server/users"), _("Users Manager"), 20).leaf = true entry({"admin", "vpn", "ipsec-server", "l2tp_user"}, cbi("ipsec-server/l2tp_user")).leaf = true - entry({"admin", "vpn", "ipsec-server", "online"}, cbi("ipsec-server/online"), _("L2TP Online Users"), 30).leaf = true entry({"admin", "vpn", "ipsec-server", "status"}, call("act_status")).leaf = true end function act_status() local e = {} - e["ipsec_status"] = luci.sys.call("/usr/bin/pgrep ipsec >/dev/null") == 0 - e["l2tp_status"] = luci.sys.call("top -bn1 | grep -v grep | grep '/var/etc/xl2tpd' >/dev/null") == 0 + e["ipsec_status"] = luci.sys.call("pidof charon >/dev/null 2>&1") == 0 luci.http.prepare_content("application/json") luci.http.write_json(e) end diff --git a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/l2tp_user.lua b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/l2tp_user.lua index 3b8460c65..e21a9043e 100644 --- a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/l2tp_user.lua +++ b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/l2tp_user.lua @@ -1,10 +1,10 @@ local d = require "luci.dispatcher" -local sys = require "luci.sys" +local fs = require "nixio.fs" m = Map("luci-app-ipsec-server", "L2TP/IPSec PSK " .. translate("Users Manager")) m.redirect = d.build_url("admin", "vpn", "ipsec-server", "users") -if sys.call("command -v xl2tpd > /dev/null") == 0 then +if fs.access("/usr/sbin/xl2tpd") then s = m:section(NamedSection, arg[1], "l2tp_users", "") s.addremove = false s.anonymous = true @@ -19,6 +19,7 @@ if sys.call("command -v xl2tpd > /dev/null") == 0 then o = s:option(Value, "password", translate("Password")) o.placeholder = translate("Password") + o.password = true o.rmempty = false o = s:option(Value, "ipaddress", translate("IP address")) diff --git a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/online.lua b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/online.lua deleted file mode 100644 index d47b30053..000000000 --- a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/online.lua +++ /dev/null @@ -1,83 +0,0 @@ -local o = require "luci.dispatcher" -local fs = require "nixio.fs" -local jsonc = require "luci.jsonc" - -local sessions = {} -local session_path = "/var/etc/xl2tpd/session" -if fs.access(session_path) then - for filename in fs.dir(session_path) do - local session_file = session_path .. "/" .. filename - local file = io.open(session_file, "r") - local t = jsonc.parse(file:read("*a")) - if t then - t.session_file = session_file - sessions[#sessions + 1] = t - end - file:close() - end -end - -local blacklist = {} -local firewall_user_path = "/etc/firewall.user" -if fs.access(firewall_user_path) then - for line in io.lines(firewall_user_path) do - local m = line:match('xl2tpd%-blacklist%-([^\n]+)') - if m then - local t = {} - t.ip = m - blacklist[#blacklist + 1] = t - end - end -end - -f = SimpleForm("processes") -f.reset = false -f.submit = false - -t = f:section(Table, sessions, translate("L2TP Online Users")) -t:option(DummyValue, "username", translate("Username")) -t:option(DummyValue, "interface", translate("Interface")) -t:option(DummyValue, "ip", translate("Client IP")) -t:option(DummyValue, "remote_ip", translate("IP address")) -t:option(DummyValue, "login_time", translate("Login Time")) - -_blacklist = t:option(Button, "_blacklist", translate("Blacklist")) -function _blacklist.render(e, t, a) - e.title = translate("Add to Blacklist") - e.inputstyle = "remove" - Button.render(e, t, a) -end -function _blacklist.write(t, s) - local e = t.map:get(s, "remote_ip") - luci.util.execi("echo 'iptables -I INPUT -s %s -p udp -m multiport --dports 500,4500,1701 -j DROP ## xl2tpd-blacklist-%s' >> /etc/firewall.user" % {e, e}) - luci.util.execi("iptables -I INPUT -s %s -p udp -m multiport --dports 500,4500,1701 -j DROP" % {e}) - luci.util.execi("rm -f " .. t.map:get(s, "session_file")) - null, t.tag_error[s] = luci.sys.process.signal(t.map:get(s, "pid"), 9) - luci.http.redirect(o.build_url("admin/vpn/ipsec-server/online")) -end - -_kill = t:option(Button, "_kill", translate("Forced offline")) -_kill.inputstyle = "remove" -function _kill.write(t, s) - luci.util.execi("rm -f " .. t.map:get(s, "session_file")) - null, t.tag_error[t] = luci.sys.process.signal(t.map:get(s, "pid"), 9) - luci.http.redirect(o.build_url("admin/vpn/ipsec-server/online")) -end - -t = f:section(Table, blacklist, translate("Blacklist")) -t:option(DummyValue, "ip", translate("IP address")) - -_blacklist2 = t:option(Button, "_blacklist2", translate("Blacklist")) -function _blacklist2.render(e, t, a) - e.title = translate("Remove from Blacklist") - e.inputstyle = "apply" - Button.render(e, t, a) -end -function _blacklist2.write(t, s) - local e = t.map:get(s, "ip") - luci.util.execi("sed -i -e '/## xl2tpd-blacklist-%s/d' /etc/firewall.user" % {e}) - luci.util.execi("iptables -D INPUT -s %s -p udp -m multiport --dports 500,4500,1701 -j DROP" % {e}) - luci.http.redirect(o.build_url("admin/vpn/ipsec-server/online")) -end - -return f diff --git a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/settings.lua b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/settings.lua index b88dd230d..52d533af5 100644 --- a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/settings.lua +++ b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/settings.lua @@ -1,5 +1,3 @@ -local sys = require "luci.sys" - m = Map("luci-app-ipsec-server", translate("IPSec VPN Server")) m.template = "ipsec-server/ipsec-server_status" @@ -9,56 +7,53 @@ s.anonymous = true o = s:option(DummyValue, "ipsec-server_status", translate("Current Condition")) o.rawhtml = true o.cfgvalue = function(t, n) - return '' + return '' .. translate("Checking...") .. '' end enabled = s:option(Flag, "enabled", translate("Enable")) -enabled.description = translate("Use a client that supports IPSec Xauth PSK (iOS or Android) to connect to this server.") +enabled.description = translate("Enable IPsec access for IKEv2 PSK, IKEv1 XAuth PSK, and IKEv2 EAP clients.") enabled.default = 0 enabled.rmempty = false clientip = s:option(Value, "clientip", translate("VPN Client IP")) -clientip.description = translate("VPN Client reserved started IP addresses with the same subnet mask, such as: 192.168.100.10/24") +clientip.description = translate("VPN client address pool in CIDR notation, such as: 10.0.10.0/24") clientip.datatype = "ip4addr" clientip.optional = false clientip.rmempty = false +clientip.placeholder = "10.0.10.0/24" + +serverip = s:option(Value, "serverip", translate("VPN Server IP")) +serverip.description = translate("VPN server address and subnet in CIDR notation, such as: 10.0.0.1/24") +serverip.datatype = "ip4addr" +serverip.optional = false +serverip.rmempty = false +serverip.placeholder = "10.0.0.1/24" + +domain = s:option(Value, "domain", translate("VPN Domain")) +domain.description = translate("Domain name used for IKEv2 certificate and Remote ID, such as: ns.miaogongzi.cc") +domain.datatype = "host" +domain.optional = false +domain.rmempty = false +domain.placeholder = "ns.miaogongzi.cc" +domain.default = "ns.miaogongzi.cc" + +certs_name = s:option(Value, "certs_name", translate("Certificate File Name")) +certs_name.description = translate("Server certificate file name under /etc/ipsec.d/certs/, such as: server.pem") +certs_name.optional = false +certs_name.rmempty = false +certs_name.placeholder = "server.pem" +certs_name.default = "server.pem" + +private_key = s:option(Value, "private_key", translate("Private Key File Name")) +private_key.description = translate("Server private key file name under /etc/ipsec.d/private/, such as: server_key.pem") +private_key.optional = false +private_key.rmempty = false +private_key.placeholder = "server_key.pem" +private_key.default = "server_key.pem" secret = s:option(Value, "secret", translate("Secret Pre-Shared Key")) +secret.description = translate("Used by IKEv2 PSK and IKEv1 XAuth PSK clients.") secret.password = true - -if sys.call("command -v xl2tpd > /dev/null") == 0 then - o = s:option(DummyValue, "l2tp_status", "L2TP " .. translate("Current Condition")) - o.rawhtml = true - o.cfgvalue = function(t, n) - return '' - end - - o = s:option(Flag, "l2tp_enable", "L2TP " .. translate("Enable")) - o.description = translate("Use a client that supports L2TP over IPSec PSK to connect to this server.") - o.default = 0 - o.rmempty = false - - o = s:option(Value, "l2tp_localip", "L2TP " .. translate("Server IP")) - o.description = translate("VPN Server IP address, such as: 192.168.101.1") - o.datatype = "ip4addr" - o.rmempty = true - o.default = "192.168.101.1" - o.placeholder = o.default - - o = s:option(Value, "l2tp_remoteip", "L2TP " .. translate("Client IP")) - o.description = translate("VPN Client IP address range, such as: 192.168.101.10-20") - o.rmempty = true - o.default = "192.168.101.10-20" - o.placeholder = o.default - - if sys.call("ls -L /usr/lib/ipsec/libipsec* 2>/dev/null >/dev/null") == 0 then - o = s:option(DummyValue, "_o", " ") - o.rawhtml = true - o.cfgvalue = function(t, n) - return string.format('%s', translate("L2TP/IPSec is not compatible with kernel-libipsec, which will disable this module.")) - end - o:depends("l2tp_enable", true) - end -end +secret.rmempty = false return m diff --git a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/users.lua b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/users.lua index 87a21be74..8f2e07315 100644 --- a/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/users.lua +++ b/luci-app-ipsec-server/luasrc/model/cbi/ipsec-server/users.lua @@ -1,10 +1,10 @@ local d = require "luci.dispatcher" -local sys = require "luci.sys" +local fs = require "nixio.fs" m = Map("luci-app-ipsec-server") -s = m:section(TypedSection, "ipsec_users", "IPSec Xauth PSK " .. translate("Users Manager")) -s.description = translate("Use a client that supports IPSec Xauth PSK (iOS or Android) to connect to this server.") +s = m:section(TypedSection, "ipsec_users", "IPSec Xauth PSK/EAP " .. translate("Users Manager")) +s.description = translate("Use a client that supports IPSec Xauth PSK or IPSec EAP (iOS, Android, macOS, or Windows) to connect to this server.") s.addremove = true s.anonymous = true s.template = "cbi/tblsection" @@ -19,9 +19,10 @@ o.rmempty = false o = s:option(Value, "password", translate("Password")) o.placeholder = translate("Password") +o.password = true o.rmempty = false -if sys.call("command -v xl2tpd > /dev/null") == 0 then +if fs.access("/usr/sbin/xl2tpd") then s = m:section(TypedSection, "l2tp_users", "L2TP/IPSec PSK " .. translate("Users Manager")) s.description = translate("Use a client that supports L2TP over IPSec PSK to connect to this server.") s.addremove = true @@ -31,7 +32,7 @@ if sys.call("command -v xl2tpd > /dev/null") == 0 then function s.create(e, t) t = TypedSection.create(e, t) luci.http.redirect(e.extedit:format(t)) - end + end o = s:option(Flag, "enabled", translate("Enabled")) o.default = 1 @@ -43,6 +44,7 @@ if sys.call("command -v xl2tpd > /dev/null") == 0 then o = s:option(Value, "password", translate("Password")) o.placeholder = translate("Password") + o.password = true o.rmempty = false o = s:option(Value, "ipaddress", translate("IP address")) diff --git a/luci-app-ipsec-server/luasrc/view/ipsec-server/ipsec-server_status.htm b/luci-app-ipsec-server/luasrc/view/ipsec-server/ipsec-server_status.htm index 93e36a405..251caacce 100644 --- a/luci-app-ipsec-server/luasrc/view/ipsec-server/ipsec-server_status.htm +++ b/luci-app-ipsec-server/luasrc/view/ipsec-server/ipsec-server_status.htm @@ -1,21 +1,61 @@ + + <% include("cbi/map") %> + diff --git a/luci-app-ipsec-server/po/zh-cn/ipsec-server.po b/luci-app-ipsec-server/po/zh-cn/ipsec-server.po index 5c9049e02..e99b576fa 100644 --- a/luci-app-ipsec-server/po/zh-cn/ipsec-server.po +++ b/luci-app-ipsec-server/po/zh-cn/ipsec-server.po @@ -1,38 +1,56 @@ msgid "IPSec VPN Server" msgstr "IPSec VPN 服务器" -msgid "Use a client that supports IPSec Xauth PSK (iOS or Android) to connect to this server." -msgstr "使用支持 IPSec Xauth PSK（iOS 或 Android）的客户端连接到此服务端。" +msgid "Enable IPsec access for IKEv2 PSK, IKEv1 XAuth PSK, and IKEv2 EAP clients." +msgstr "启用 IKEv2 PSK、IKEv1 XAuth PSK 和 IKEv2 EAP 客户端接入。" -msgid "Use a client that supports L2TP over IPSec PSK to connect to this server." -msgstr "使用支持 L2TP over IPSec PSK 的客户端连接到此服务端。" +msgid "Use a client that supports IPSec Xauth PSK or IPSec EAP (iOS, Android, macOS, or Windows) to connect to this server." +msgstr "请使用支持 IPSec Xauth PSK 或 IPSec EAP 的客户端（iOS、Android、macOS、Windows）连接到此服务器。" msgid "Current Condition" msgstr "当前状态" -msgid "General settings" +msgid "Checking..." +msgstr "正在检查..." + +msgid "Status unavailable" +msgstr "状态不可用" + +msgid "General Settings" msgstr "基本设置" +msgid "Enable" +msgstr "启用" + msgid "Enabled" msgstr "启用" msgid "VPN Client IP" -msgstr "VPN客户端地址段" +msgstr "VPN 客户端地址池" -msgid "VPN Client reserved started IP addresses with the same subnet mask, such as: 192.168.100.10/24" -msgstr "VPN客户端获取IP的起始地址，例如：192.168.100.10/24" +msgid "VPN Server IP" +msgstr "VPN 服务端地址" + +msgid "VPN client address pool in CIDR notation, such as: 10.0.10.0/24" +msgstr "VPN 客户端地址池（CIDR 格式），例如：10.0.10.0/24" + +msgid "VPN server address and subnet in CIDR notation, such as: 10.0.0.1/24" +msgstr "VPN 服务端地址和子网（CIDR 格式），例如：10.0.0.1/24" + +msgid "Domain name used for IKEv2 certificate and Remote ID, such as: ns.miaogongzi.cc" +msgstr "用于 IKEv2 证书和远程 ID 的域名，例如：ns.miaogongzi.cc" + +msgid "Server private key file name under /etc/ipsec.d/private/, such as: server_key.pem" +msgstr "位于 /etc/ipsec.d/private/ 的服务器私钥文件名，例如：server_key.pem" + +msgid "Server certificate file name under /etc/ipsec.d/certs/, such as: server.pem" +msgstr "位于 /etc/ipsec.d/certs/ 的服务器证书文件名，例如：server.pem" msgid "Secret Pre-Shared Key" -msgstr "PSK密钥" +msgstr "PSK 密钥" -msgid "VPN Server IP address, such as: 192.168.101.1" -msgstr "VPN服务端IP地址，例如：192.168.101.1" - -msgid "VPN Client IP address range, such as: 192.168.101.10-20" -msgstr "VPN客户端获取IP范围，例如：192.168.101.10-20" - -msgid "L2TP/IPSec is not compatible with kernel-libipsec, which will disable this module." -msgstr "L2TP/IPSec不兼容kernel-libipsec，开启将会禁用此模块。" +msgid "Used by IKEv2 PSK and IKEv1 XAuth PSK clients." +msgstr "供 IKEv2 PSK 和 IKEv1 XAuth PSK 客户端使用。" msgid "Users Manager" msgstr "用户管理" @@ -49,29 +67,23 @@ msgstr "IP 地址" msgid "Automatically" msgstr "自动分配" -msgid "Online Users" -msgstr "在线用户" +msgid "Static Routes" +msgstr "静态路由" -msgid "L2TP Online Users" -msgstr "L2TP 在线用户" - -msgid "Login Time" -msgstr "登录时间" - -msgid "Blacklist" -msgstr "黑名单" - -msgid "Add to Blacklist" -msgstr "加入黑名单" - -msgid "Remove from Blacklist" -msgstr "移出黑名单" - -msgid "Forced offline" -msgstr "强制下线" +msgid "Use a client that supports L2TP over IPSec PSK to connect to this server." +msgstr "使用支持 L2TP over IPSec PSK 的客户端连接到此服务器。" msgid "NOT RUNNING" msgstr "未运行" msgid "RUNNING" msgstr "运行中" + +msgid "Certificate File Name" +msgstr "证书文件名" + +msgid "Private Key File Name" +msgstr "私钥文件名" + +msgid "VPN Domain" +msgstr "VPN 域名" diff --git a/luci-app-ipsec-server/root/etc/config/luci-app-ipsec-server b/luci-app-ipsec-server/root/etc/config/luci-app-ipsec-server index 6d90a5d69..fcfa85bb5 100644 --- a/luci-app-ipsec-server/root/etc/config/luci-app-ipsec-server +++ b/luci-app-ipsec-server/root/etc/config/luci-app-ipsec-server @@ -2,6 +2,9 @@ config service 'ipsec' option enabled '0' option secret 'ipsec' - option clientip '192.168.100.10/24' - + option clientip '10.0.10.0/24' + option serverip '10.0.0.1/24' + option certs_name 'server.pem' + option private_key 'server_key.pem' + option domain 'ns.miaogongzi.cc' diff --git a/luci-app-ipsec-server/root/etc/init.d/luci-app-ipsec-server b/luci-app-ipsec-server/root/etc/init.d/luci-app-ipsec-server index 7b7e0a235..5164e99ee 100755 --- a/luci-app-ipsec-server/root/etc/init.d/luci-app-ipsec-server +++ b/luci-app-ipsec-server/root/etc/init.d/luci-app-ipsec-server @@ -13,6 +13,10 @@ L2TP_OPTIONS_FILE=${L2TP_PATH}/options.xl2tpd L2TP_LOG_FILE=${L2TP_PATH}/xl2tpd.log vt_clientip=$(uci -q get ${CONFIG}.@service[0].clientip) +vt_serverip=$(uci -q get ${CONFIG}.@service[0].serverip) +certs_name=$(uci -q get ${CONFIG}.@service[0].certs_name) +private_key=$(uci -q get ${CONFIG}.@service[0].private_key) +vpn_domain=$(uci -q get ${CONFIG}.@service[0].domain) l2tp_enabled=$(uci -q get ${CONFIG}.@service[0].l2tp_enable) l2tp_localip=$(uci -q get ${CONFIG}.@service[0].l2tp_localip) @@ -24,8 +28,8 @@ get_enabled_anonymous_secs() { ipt_rule() { if [ "$1" = "add" ]; then - iptables -t nat -I POSTROUTING -s ${vt_clientip} -m comment --comment "${ipt_flag}" -j MASQUERADE 2>/dev/null - iptables -I forwarding_rule -s ${vt_clientip} -m comment --comment "${ipt_flag}" -j ACCEPT 2>/dev/null + iptables -t nat -I POSTROUTING -s ${vt_serverip} -m comment --comment "${ipt_flag}" -j MASQUERADE 2>/dev/null + iptables -I forwarding_rule -s ${vt_serverip} -m comment --comment "${ipt_flag}" -j ACCEPT 2>/dev/null iptables -I forwarding_rule -m policy --dir in --pol ipsec --proto esp -m comment --comment "${ipt_flag}" -j ACCEPT 2>/dev/null iptables -I forwarding_rule -m policy --dir out --pol ipsec --proto esp -m comment --comment "${ipt_flag}" -j ACCEPT 2>/dev/null iptables -I INPUT -p udp -m multiport --dports 500,4500 -m comment --comment "${ipt_flag}" -j ACCEPT 2>/dev/null @@ -70,9 +74,9 @@ gen_include() { start() { local vt_enabled=$(uci -q get ${CONFIG}.@service[0].enabled) - [ "$vt_enabled" = 0 ] && return 1 + [ "$vt_enabled" = 1 ] || return 0 - local vt_gateway="${vt_clientip%.*}.1" + local vt_gateway="${vt_serverip%.*}.1" local vt_secret=$(uci -q get ${CONFIG}.@service[0].secret) local l2tp_enabled=$(uci -q get ${CONFIG}.@service[0].l2tp_enable) @@ -185,8 +189,8 @@ start() { conn Non-L2TP leftsubnet=0.0.0.0/0 - rightsubnet=${vt_clientip} - rightsourceip=${vt_clientip} + rightsubnet=${vt_serverip} + rightsourceip=${vt_serverip} rightdns=${vt_gateway} ike=aes128-sha1-modp2048,aes128-sha1-modp1024,3des-sha1-modp1024,3des-sha1-modp1536 esp=aes128-sha1,3des-sha1 @@ -198,6 +202,52 @@ start() { leftauth=psk rightauth=psk rightauth2=xauth + + conn IKEv2-PSK + keyexchange=ikev2 + auto=add + left=%defaultroute + leftauth=psk + leftsubnet=0.0.0.0/0 + right=%any + rightauth=psk + rightsourceip=${vt_clientip} + + ike=aes128-sha1-modp2048,aes128-sha1-modp1024,3des-sha1-modp1024,3des-sha1-modp1536 + esp=aes128-sha1,3des-sha1 + dpdaction=clear + dpddelay=30s + rekey=no + compress=no + + conn ikev2-EAP + keyexchange=ikev2 + auto=add + fragmentation=yes + forceencaps=yes + + left=%defaultroute + leftid=@$vpn_domain + leftauth=pubkey + leftcert=${certs_name} + leftsendcert=always + leftsubnet=0.0.0.0/0 + + right=%any + rightid=%any + rightauth=eap-mschapv2 + eap_identity=%identity + rightsourceip=${vt_clientip} + rightdns=10.0.0.1 + + ike=aes256-sha256-modp1024,aes256-sha1-modp1024,aes128-sha256-modp1024,aes128-sha1-modp1024,aes256-sha256-modp2048,aes256-sha1-modp2048 + esp=aes256-sha256,aes128-sha256,aes256-sha1,aes128-sha1 + dpdaction=clear + dpddelay=30s + rekey=no + compress=no + + $ipsec_l2tp_config EOF @@ -205,6 +255,7 @@ start() { cat > /etc/ipsec.secrets <<-EOF # /etc/ipsec.secrets - strongSwan IPsec secrets file : PSK "$vt_secret" + : RSA /etc/ipsec.d/private/$private_key EOF local ipsec_users=$(get_enabled_anonymous_secs "@ipsec_users") @@ -220,6 +271,7 @@ start() { [ -n "${u_password}" ] || continue echo "${u_username} : XAUTH '${u_password}'" >> ${IPSEC_SECRETS_FILE} + echo "${u_username} : EAP '${u_password}'" >> ${IPSEC_SECRETS_FILE} done } unset user @@ -230,7 +282,7 @@ start() { gen_include uci -q batch <<-EOF >/dev/null - set network.ipsec_server.ipaddr="${vt_clientip%.*}.1" + set network.ipsec_server.ipaddr="${vt_serverip%.*}.1" commit network EOF ifup ipsec_server > /dev/null 2>&1 @@ -254,7 +306,7 @@ gen_iface_and_firewall() { set network.ipsec_server.device="ipsec0" set network.ipsec_server.device="ipsec0" set network.ipsec_server.proto="static" - set network.ipsec_server.ipaddr="${vt_clientip%.*}.1" + set network.ipsec_server.ipaddr="${vt_serverip%.*}.1" set network.ipsec_server.netmask="255.255.255.0" commit network diff --git a/luci-app-ipsec-server/root/etc/uci-defaults/luci-app-ipsec-server b/luci-app-ipsec-server/root/etc/uci-defaults/luci-app-ipsec-server index d0e598e94..c22e3b498 100644 --- a/luci-app-ipsec-server/root/etc/uci-defaults/luci-app-ipsec-server +++ b/luci-app-ipsec-server/root/etc/uci-defaults/luci-app-ipsec-server @@ -7,6 +7,7 @@ } EEOF } +rm -f /usr/share/ucitrack/luci-app-luci-app-ipsec-server.json uci -q batch <<-EOF >/dev/null delete firewall.luci_app_ipsec_server @@ -14,6 +15,7 @@ uci -q batch <<-EOF >/dev/null set firewall.luci_app_ipsec_server.type=script set firewall.luci_app_ipsec_server.path=/var/etc/ipsecvpn.include set firewall.luci_app_ipsec_server.reload=1 + commit firewall EOF uci -q batch <<-EOF >/dev/null diff --git a/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-down b/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-down deleted file mode 100755 index 9434e7615..000000000 --- a/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-down +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh - -_LOGOUT_TIME="$(date "+%Y-%m-%d %H:%M:%S")" -CONFIG="luci-app-ipsec-server" -L2TP_PATH=/var/etc/xl2tpd -L2TP_SESSION_PATH=${L2TP_PATH}/session - -_USERNAME=${PEERNAME} -_IFACE=${1} -_TTY=${2} -_SPEED=${3} -_LOCALIP=${4} -_PEERIP=${5} -_REMOTEIP=${6} -_BYTES_SENT=${BYTES_SENT} -_BYTES_RCVD=${BYTES_RCVD} -_CONNECT_TIME=${CONNECT_TIME} - -rm -f ${L2TP_SESSION_PATH}/${_USERNAME}.${_IFACE} -rm -f /var/run/${_IFACE}.pid - -#可根据退出的账号自定义脚本，如静态路由表，组网等。 -SCRIPT="/usr/share/xl2tpd/ip-down.d/${_USERNAME}" -[ -s "$SCRIPT" ] && { - [ ! -x "$SCRIPT" ] && chmod 0755 "$SCRIPT" - "$SCRIPT" "$@" -} diff --git a/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-up b/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-up deleted file mode 100755 index 6109d037e..000000000 --- a/luci-app-ipsec-server/root/usr/share/xl2tpd/ip-up +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/sh - -_LOGIN_TIME="$(date "+%Y-%m-%d %H:%M:%S")" -CONFIG="luci-app-ipsec-server" -L2TP_PATH=/var/etc/xl2tpd -L2TP_SESSION_PATH=${L2TP_PATH}/session - -_USERNAME=${PEERNAME} -_IFACE=${1} -_TTY=${2} -_SPEED=${3} -_LOCALIP=${4} -_PEERIP=${5} - -_PID=$(cat /var/run/${_IFACE}.pid 2>/dev/null) -_REMOTEIP=$(cat /var/etc/xl2tpd/xl2tpd.log 2>/dev/null | grep "PID: ${_PID}" | grep -o -E '([0-9]{1,3}[\.]){3}[0-9]{1,3}') - -mkdir -p ${L2TP_SESSION_PATH} - -cat <<-EOF > ${L2TP_SESSION_PATH}/${_USERNAME}.${_IFACE} - { - "username": "${_USERNAME}", - "interface": "${_IFACE}", - "tty": "${_TTY}", - "speed": "${_SPEED}", - "ip": "${_PEERIP}", - "remote_ip": "${_REMOTEIP}", - "pid": "${_PID}", - "login_time": "${_LOGIN_TIME}" - } -EOF - -#只能单用户使用 -cfgid=$(uci show ${CONFIG} | grep "@l2tp_users" | grep "\.username='${_USERNAME}'" | cut -d '.' -sf 2) -[ -n "$cfgid" ] && { - HAS_LOGIN=$(ls ${L2TP_SESSION_PATH} | grep "^${_USERNAME}\.ppp" | grep -v "${_IFACE}") - [ -n "$HAS_LOGIN" ] && { - #踢出之前的用户 - KO_IFACE=$(echo $HAS_LOGIN | awk -F '.' '{print $2}') - KO_PID=$(cat /var/run/${KO_IFACE}.pid 2>/dev/null) - [ -n "$KO_PID" ] && kill -9 ${KO_PID} >/dev/null 2>&1 - rm -f ${L2TP_SESSION_PATH}/${HAS_LOGIN} - rm -f /var/run/${KO_IFACE}.pid - } - routes=$(uci -q get ${CONFIG}.${cfgid}.routes) - [ -n "$routes" ] && { - for router in ${routes}; do - route add -net ${router} dev ${_IFACE} >/dev/null 2>&1 - done - } -} - -#可根据登录的账号自定义脚本，如组网、日志、限速、权限等特殊待遇。 -SCRIPT="/usr/share/xl2tpd/ip-up.d/${_USERNAME}" -[ -s "$SCRIPT" ] && { - [ ! -x "$SCRIPT" ] && chmod 0755 "$SCRIPT" - "$SCRIPT" "$@" -} diff --git a/mwan3-nft/Makefile b/mwan3-nft/Makefile index 3dbf9d4eb..d6cf5068d 100644 --- a/mwan3-nft/Makefile +++ b/mwan3-nft/Makefile @@ -8,8 +8,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=mwan3-nft -PKG_VERSION:=3.6.9 -PKG_RELEASE:=3 +PKG_VERSION:=3.6.12 +PKG_RELEASE:=4 PKG_MAINTAINER:=Florian Eckert PKG_LICENSE:=GPL-2.0 @@ -35,7 +35,8 @@ define Package/$(PKG_NAME) +ucode-mod-uci \ +ucode-mod-ubus \ +ucode-mod-fs \ - +ucode-mod-socket + +ucode-mod-socket \ + +ucode-mod-log CONFLICTS:=mwan3 TITLE:=Multiwan hotplug script with connection tracking support (ucode rtmon) MAINTAINER:=Florian Eckert @@ -220,6 +221,10 @@ define Package/$(PKG_NAME)/install $(INSTALL_BIN) ./files/usr/share/rpcd/ucode/mwan3 \ $(1)/usr/share/rpcd/ucode/ + $(INSTALL_DIR) $(1)/usr/share/ucode/mwan3 + $(INSTALL_DATA) ./files/usr/share/ucode/mwan3/common.uc \ + $(1)/usr/share/ucode/mwan3/ + $(INSTALL_DIR) $(1)/usr/sbin $(INSTALL_BIN) ./files/usr/sbin/mwan3 \ $(1)/usr/sbin/ diff --git a/mwan3-nft/README.md b/mwan3-nft/README.md index a0451e2cf..41fd52406 100644 --- a/mwan3-nft/README.md +++ b/mwan3-nft/README.md @@ -1,5 +1,5 @@ # mwan3 nftables User and Developer Reference -### mwan3 version: 3.6.9 +### mwan3 version: 3.6.12 Covers the nftables port of the mwan3 multi-WAN policy routing framework. --- @@ -20,40 +20,45 @@ Covers the nftables port of the mwan3 multi-WAN policy routing framework. 6. [File Reference](#6-file-reference) - 6.1 [mwan3-skeleton.nft](#61-libmwan3mwan3-skeletonnft-static) - 6.2 [common.sh](#62-libmwan3commonsh) - - 6.3 [mwan3.sh](#63-libmwan3mwan3sh) - - 6.4 [init.d/mwan3](#64-etcinitdmwan3) - - 6.5 [25-mwan3 - hotplug](#65-etchotplugdiface25-mwan3) - - 6.6 [usr/sbin/mwan3 - CLI](#66-usrsbinmwan3-cli) - - 6.7 [mwan3rtmon](#67-usrsbinmwan3rtmon) - - 6.8 [rpcd/ucode/mwan3](#68-usrsharerpcducodemwan3) - - 6.9 [Makefile](#69-makefile) - - 6.10 [mwan3track](#610-usrsbinmwan3track) - - 6.11 [mwan3-lb-test](#611-usrsbinmwan3-lb-test) - - 6.12 [mwan3-diag](#612-usrsbinmwan3-diag) - - 6.13 [mwan3ct](#613-usrsbinmwan3ct) - - 6.14 [mwan3-get-addr.uc](#614-libmwan3mwan3-get-addruc) - - 6.15 [mwan3-manage-rules.uc](#615-libmwan3mwan3-manage-rulesuc) - - 6.16 [mwan3-list-routes.uc](#616-libmwan3mwan3-list-routesuc) - - 6.17 [mwan3-create-iface-route.uc](#617-libmwan3mwan3-create-iface-routeuc) - - 6.18 [mwan3ipcheck](#618-usrbinmwan3ipcheck) + - 6.3 [common.uc](#63-usrshareucodemwan3commonuc) + - 6.4 [mwan3.sh](#64-libmwan3mwan3sh) + - 6.5 [init.d/mwan3](#65-etcinitdmwan3) + - 6.6 [25-mwan3 - hotplug](#66-etchotplugdiface25-mwan3) + - 6.7 [usr/sbin/mwan3 - CLI](#67-usrsbinmwan3-cli) + - 6.8 [mwan3rtmon](#68-usrsbinmwan3rtmon) + - 6.9 [rpcd/ucode/mwan3](#69-usrsharerpcducodemwan3) + - 6.10 [Makefile](#610-makefile) + - 6.11 [mwan3track](#611-usrsbinmwan3track) + - 6.12 [mwan3-lb-test](#612-usrsbinmwan3-lb-test) + - 6.13 [mwan3-diag](#613-usrsbinmwan3-diag) + - 6.14 [mwan3ct](#614-usrsbinmwan3ct) + - 6.15 [mwan3-get-addr.uc](#615-libmwan3mwan3-get-addruc) + - 6.16 [mwan3-manage-rules.uc](#616-libmwan3mwan3-manage-rulesuc) + - 6.17 [mwan3-list-routes.uc](#617-libmwan3mwan3-list-routesuc) + - 6.18 [mwan3-create-iface-route.uc](#618-libmwan3mwan3-create-iface-routeuc) + - 6.19 [mwan3ipcheck](#619-usrbinmwan3ipcheck) 7. [Function Reference](#7-function-reference) - 7.1 [common.sh Functions](#71-commonsh-functions) - - 7.2 [Set Management Functions](#72-set-management-functions) - - 7.3 [General Rule Setup](#73-general-rule-setup) - - 7.4 [Interface Management](#74-interface-management) - - 7.5 [Policy & Load Balancing](#75-policy--load-balancing) - - 7.6 [Sticky Routing](#76-sticky-routing) - - 7.7 [User Rules](#77-user-rules) - - 7.8 [User-defined nft Set Management](#78-user-defined-nft-set-management) - - 7.9 [Status Reporting](#79-status-reporting) - - 7.10 [Lifecycle & Hotplug](#710-lifecycle--hotplug) + - 7.2 [common.uc Functions](#72-commonuc-functions) + - 7.3 [mwan3.sh Functions](#73-mwan3sh-functions) + - 7.3.1 [Set Management Functions](#731-set-management-functions) + - 7.3.2 [General Rule Setup](#732-general-rule-setup) + - 7.3.3 [Interface Management](#733-interface-management) + - 7.3.4 [Policy & Load Balancing](#734-policy--load-balancing) + - 7.3.5 [Sticky Routing](#735-sticky-routing) + - 7.3.6 [User Rules](#736-user-rules) + - 7.3.7 [User-defined nft Set Management](#737-user-defined-nft-set-management) + - 7.3.8 [Status Reporting](#738-status-reporting) + - 7.3.9 [Lifecycle & Hotplug](#739-lifecycle--hotplug) + - 7.4 [mwan3rtmon Functions](#74-mwan3rtmon-functions) + - 7.5 [mwan3track Functions](#75-mwan3track-functions) 8. [Load Balancing with numgen](#8-load-balancing-with-numgen) 9. [Sticky Routing Detail](#9-sticky-routing-detail) 10. [Service Lifecycle and Conntrack Management](#10-service-lifecycle-and-conntrack-management) - 10.1 [Start](#101-start) - 10.2 [Reload](#102-reload) - - 10.3 [Interface Up (hotplug)](#103-interface-up-hotplug) - - 10.4 [Interface Down (hotplug)](#104-interface-down-hotplug) + - 10.3 [Interface Up and Tracker Recovery (hotplug)](#103-interface-up-and-tracker-recovery-hotplug) + - 10.4 [Interface Down and Soft Failure (hotplug)](#104-interface-down-and-soft-failure-hotplug) - 10.5 [Stop](#105-stop) - 10.6 [Conntrack Management](#106-conntrack-management) 11. [Atomic Non-destructive Reload](#11-atomic-non-destructive-reload) @@ -85,36 +90,39 @@ Covers the nftables port of the mwan3 multi-WAN policy routing framework. - 17.1 [mwan3-lb-test: Load Balancing Distribution Verifier](#171-mwan3-lb-test-load-balancing-distribution-verifier) - 17.2 [mwan3-diag: Network Diagnostic Report](#172-mwan3-diag-network-diagnostic-report) 18. [Changelog](#18-changelog) - - 18.1 [Version 3.6.9](#181-version-369) - - 18.2 [Version 3.6.8](#182-version-368) - - 18.3 [Version 3.6.7](#183-version-367) - - 18.4 [Version 3.6.6](#184-version-366) - - 18.5 [Version 3.6.5](#185-version-365) - - 18.6 [Version 3.6.4](#186-version-364) - - 18.7 [Version 3.6.3](#187-version-363) - - 18.8 [Version 3.6.2](#188-version-362) - - 18.9 [Version 3.6.1](#189-version-361) - - 18.10 [Version 3.6](#1810-version-36) - - 18.11 [Version 3.5.3](#1811-version-353) - - 18.12 [Version 3.5.2](#1812-version-352) - - 18.13 [Version 3.5.1](#1813-version-351) - - 18.14 [Version 3.5](#1814-version-35) - - 18.15 [Version 3.4.1 (Unreleased)](#1815-version-341-unreleased) - - 18.16 [Version 3.4](#1816-version-34) - - 18.17 [Version 3.3.5](#1817-version-335) - - 18.18 [Version 3.3.4](#1818-version-334) - - 18.19 [Version 3.3.3](#1819-version-333) - - 18.20 [Version 3.3.2](#1820-version-332) - - 18.21 [Version 3.3.1](#1821-version-331) - - 18.22 [Version 3.3](#1822-version-33) - - 18.23 [Version 3.2.3](#1823-version-323) - - 18.24 [Version 3.2.2](#1824-version-322) - - 18.25 [Version 3.2.1](#1825-version-321) - - 18.26 [Version 3.2](#1826-version-32) - - 18.27 [Version 3.1.4](#1827-version-314) - - 18.28 [Version 3.1.3](#1828-version-313) - - 18.29 [Version 3.1.2](#1829-version-312) - - 18.30 [Version 3.1.1](#1830-version-311) + - 18.1 [Version 3.6.12](#181-version-3612) + - 18.2 [Version 3.6.11](#182-version-3611) + - 18.3 [Version 3.6.10](#183-version-3610) + - 18.4 [Version 3.6.9](#184-version-369) + - 18.5 [Version 3.6.8](#185-version-368) + - 18.6 [Version 3.6.7](#186-version-367) + - 18.7 [Version 3.6.6](#187-version-366) + - 18.8 [Version 3.6.5](#188-version-365) + - 18.9 [Version 3.6.4](#189-version-364) + - 18.10 [Version 3.6.3](#1810-version-363) + - 18.11 [Version 3.6.2](#1811-version-362) + - 18.12 [Version 3.6.1](#1812-version-361) + - 18.13 [Version 3.6](#1813-version-36) + - 18.14 [Version 3.5.3](#1814-version-353) + - 18.15 [Version 3.5.2](#1815-version-352) + - 18.16 [Version 3.5.1](#1816-version-351) + - 18.17 [Version 3.5](#1817-version-35) + - 18.18 [Version 3.4.1 (Unreleased)](#1818-version-341-unreleased) + - 18.19 [Version 3.4](#1819-version-34) + - 18.20 [Version 3.3.5](#1820-version-335) + - 18.21 [Version 3.3.4](#1821-version-334) + - 18.22 [Version 3.3.3](#1822-version-333) + - 18.23 [Version 3.3.2](#1823-version-332) + - 18.24 [Version 3.3.1](#1824-version-331) + - 18.25 [Version 3.3](#1825-version-33) + - 18.26 [Version 3.2.3](#1826-version-323) + - 18.27 [Version 3.2.2](#1827-version-322) + - 18.28 [Version 3.2.1](#1828-version-321) + - 18.29 [Version 3.2](#1829-version-32) + - 18.30 [Version 3.1.4](#1830-version-314) + - 18.31 [Version 3.1.3](#1831-version-313) + - 18.32 [Version 3.1.2](#1832-version-312) + - 18.33 [Version 3.1.1](#1833-version-311) 19. [Specific use-cases](#19-specific-use-cases) - 19.1 [Tailscale](#191-tailscale) @@ -399,6 +407,10 @@ These chains are jumped from `mwan3_prerouting` and `mwan3_output` before the us **Source match - `mwan3_iface_in_*` chains:** Each per-interface chain includes source-match rules against all three set groups (`ip saddr @set`, scoped to the interface's address family). If an inbound WAN packet's source address is in any of the sets, it is stamped `MMX_DEFAULT`. Traffic arriving on a WAN interface from a connected or bypassed address does not get the WAN's interface mark, so replies to it use the main routing table rather than the WAN-specific policy route. +Routes wide enough to be a component of a default route are never swept into the connected or custom sets. A destination prefix of `/2` or shorter for IPv4, or `/3` or shorter for IPv6, is treated as default-equivalent and excluded by every producer that fills those sets: the startup seeding in `mwan3_set_connected_sets()` and `mwan3_set_custom_sets()`, and each of `mwan3rtmon`'s three sweeps. Each route is judged on its own, so a client that has installed only half of a `0.0.0.0/1` plus `128.0.0.0/1` pair is excluded just the same. The `224.0.0.0/3` element of `mwan3_connected_v4` is unaffected: it is added directly rather than swept from a route, and the IPv4 threshold stops at `/2`. + +Without this exclusion a single such route would silently disable mwan3 for its whole address family. The bypass chains stamp `MMX_DEFAULT` before the user rules and again after them, so a covering route in the connected set would override rule steering, sticky pinning and a fail-closed `last_resort` policy used as a kill switch, and the source-match rules in the `mwan3_iface_in_*` chains would strip inbound reply-path classification from every WAN at once. No genuine network route is as wide as these thresholds. A deliberately broad bypass is still available, but it has to be asked for explicitly through `globals.bypass_network`, which populates the `mwan3_dynamic` sets and is never swept from the routing table. + ### Per-Interface Chain Detail Each `mwan3_iface_in_` chain handles packets arriving on a specific WAN device: @@ -462,9 +474,17 @@ Each per-interface routing table is built by `mwan3_create_iface_route()` from t The `MWAN3_ROUTE_LINE_EXP` sed expression strips transient attributes (`linkdown`, `offload`, expiry timestamps, `error` codes) before copying routes. Routes whose device matches the mwan3 interface's network device keep their original `dev` clause; the interface's default gateway (`default via ... dev ...`) is the critical entry that directs all policy-routed traffic to the correct WAN path. -### 5.2 Three ip Rule Tiers Per Interface +Every mwan3-managed interface must carry a literal default route out of its own device in its own per-interface table: `0.0.0.0/0` for an IPv4 interface, `::/0` for an IPv6 one. This is a requirement rather than a recommendation. mwan3 is a multi-WAN manager and its interfaces are by definition whole-internet paths that policies choose between, so an interface that cannot act as a default route cannot do the job mwan3 exists to give it. The requirement is capability, not use: the interface must be able to carry the whole internet even if no rule ever steers everything to it. For a WireGuard interface this means `allowed_ips '0.0.0.0/0'` (and `::/0` for an IPv6 interface) together with `route_allowed_ips '1'`, which installs a literal default route on the interface. -`mwan3_create_iface_rules()` installs three `ip rule` entries for each mwan3 interface. For IPv4 interfaces these are installed via `ip rule`; for IPv6 via `ip -6 rule`. All three tiers are installed together when the interface comes up (`ifup` hotplug event) and removed together when it goes down (`ifdown`). +An interface whose own table holds no literal default route out of its own device is taken offline, deliberately. The route monitor evaluates each tracked interface's table and writes the verdict to the interface's `ROUTE_STATUS` file, logging the loss at warn level as ` default route down`. The tracker reads that file once per round and treats `down` as a failed round, so the ordinary score and hysteresis carry the interface offline, and back online when the route returns. An interface that reads offline with that message in the log is not misbehaving; it is this requirement being enforced, and the remedy is to give the interface a default route. + +Some VPN clients install routes that cover the whole internet without being a default route, and the shape is often the provider's choice rather than yours. OpenVPN's `redirect-gateway def1` installs `0.0.0.0/1` and `128.0.0.0/1` in place of a default, split-tunnel WireGuard configurations do the same, and some IPv6 providers push the global-unicast prefix `2000::/3`. None of these is a literal default route, so none of them satisfies the requirement. Where the client configuration is under your control, prefer plain `redirect-gateway` to `redirect-gateway def1`, which installs a genuine default. Where it is not, such as a server-pushed `def1` pair or a server-pushed `2000::/3`, add a literal default route to the interface yourself as a static route in the interface's configuration; the pushed routes are left untouched and the interface then satisfies the route-state check. + +### 5.2 Four ip Rule Tiers Per Interface + +`mwan3_create_iface_rules()` installs the Tier 1, Tier 2 and Tier 4 `ip rule` entries for each mwan3 interface. For IPv4 interfaces these are installed via `ip rule`; for IPv6 via `ip -6 rule`, with Tier 4 installed for IPv4 interfaces only. All three are installed when the interface comes up (`ifup` or `connected` hotplug event) and removed when it goes down (`ifdown`). + +Tier 3 (the per-WAN unreachable backstop) has a separate lifecycle managed by `mwan3_set_member_backstops()`. One Tier 3 rule is installed for every configured interface at service start and on every reload, and removed only when the service stops. The Tier 3 rule is therefore present for all configured WANs, including those that are currently offline or disabled, for as long as the service is running. **Tier 1 - iif lookup** (priority `id + iif_rule_base`, default `id + 1000`): @@ -490,6 +510,18 @@ ip rule add pref fwmark / unreachable Matches the same fwmark as Tier 2 but returns ICMP unreachable instead of routing. Sits below Tier 2 in priority (higher priority number = lower precedence). Serves as a safety net: if the Tier 2 lookup finds an empty or incomplete routing table (e.g. the default route is transiently absent during route installation), the packet receives an explicit unreachable response instead of silently falling through to the main routing table and potentially leaking out a different WAN. +**Tier 4 - source address lookup** (priority `id + unreachable_rule_base + MWAN3_INTERFACE_MAX + 1`, default `id + 3061`, IPv4 only): + +``` +ip rule add pref from
lookup +``` + +Matches router-originated traffic by its source address rather than by mark or by arrival device. `mwan3_create_iface_rules()` delegates to `mwan3-manage-rules.uc add-src`, which dumps the interface's addresses and installs one rule per global-scope IPv4 address carried on the device, all at the same priority, so a WAN holding several addresses gets several rules. + +The tier exists because a sender that performs its own route lookup outside netfilter's view never acquires an mwan3 mark, so Tier 2 cannot steer it. Its packets follow the main table out of the default WAN even though the source address they carry belongs to a different WAN, and are dropped upstream by BCP38 filtering or masqueraded to the wrong address. WireGuard is the common case: a handshake arriving on a non-default WAN is answered from that WAN's address but leaves by the default WAN, so the handshake never completes. Any locally originated flow that no user rule classifies shares the gap. With the source rule in place the route lookup resolves back to the arrival WAN's own table and the reply keeps a source address that is valid on the path it takes. + +The tier sits below Tier 3 in priority, so marked traffic still matches Tier 2 or the Tier 3 backstop first and policy verdicts and the fail-closed backstops behave exactly as they did without it. The rules are inert for forwarded traffic, whose source is a downstream address, and for a single-WAN setup. + ### 5.3 Global Policy Rules `mwan3_set_general_rules()` installs two rules that are not tied to any specific interface. These handle the mark values reserved for policy `last_resort` actions: @@ -525,6 +557,9 @@ The three rule-base offsets are configurable in UCI `config globals`: | `iif_rule_base` | 1000 | 1001 - 1060 (one per interface) | | `fwmark_rule_base` | 2000 | 2001 - 2062 (per-interface + 2 global) | | `unreachable_rule_base` | 3000 | 3001 - 3060 (one per interface) | +| none, derived | 3061 | 3062 - 3121 (Tier 4, one or more per IPv4 interface) | + +The Tier 4 source band has no UCI option of its own. It is derived as `unreachable_rule_base + MWAN3_INTERFACE_MAX + 1 + id`, so it moves with `unreachable_rule_base` and always sits immediately above the unreachable band, separated from it by one unused priority (3061 with the defaults). Two ordering constraints must hold at startup: @@ -535,10 +570,12 @@ fwmark_rule_base + MWAN3_INTERFACE_MAX + 1 < unreachable_rule_base The `+ 1` in the second constraint accounts for the global blackhole and unreachable entries at the top of the fwmark tier (`MM_BLACKHOLE + fwmark_rule_base` and `MM_UNREACHABLE + fwmark_rule_base`). If either constraint is violated, `mwan3_init` logs a warning and reverts all three to 1000/2000/3000. -`mwan3_delete_face_rules()` uses content-based matching to find and delete rules: it scans `ip rule list` for entries referencing `lookup ` to identify iif rules, and derives the fwmark value from the fwmark lookup rule before deleting the matching unreachable rule. This approach remains correct if the bases or `MMX_MASK` are changed between service restarts. +`mwan3_delete_iface_rules()` removes the Tier 1, Tier 2 and Tier 4 rules by delegating to `mwan3-manage-rules.uc delete-iface`, which locates each rule by its priority offset and table ID and removes it via RTM_DELRULE; a Tier 4 rule is identified by carrying a source address at the source-band priority, so every address rule the interface holds goes in the one pass. This approach remains correct if the bases or `MMX_MASK` are changed between service restarts. The Tier 3 unreachable backstop is not deleted here; it persists until service stop. ### 5.6 Example: Live ip rule Output +Three WANs are configured on this router: `eth1` (id 1) and `eth2` (id 2) are online, while a third interface (id 3) is configured but currently offline. + ``` 0: from all lookup local 1001: from all iif eth1 lookup 1 @@ -549,10 +586,17 @@ The `+ 1` in the second constraint accounts for the global blackhole and unreach 2062: from all fwmark 0x3e00/0x3f00 unreachable 3001: from all fwmark 0x100/0x3f00 unreachable 3002: from all fwmark 0x200/0x3f00 unreachable +3003: from all fwmark 0x300/0x3f00 unreachable +3062: from 192.0.2.2 lookup 1 +3063: from 198.51.100.2 lookup 2 32766: from all lookup main 32767: from all lookup default ``` +The two entries at `3062` and `3063` are the Tier 4 source rules, one per global-scope IPv4 address on each online WAN's device. Router-originated traffic that carries no mwan3 mark, such as a WireGuard reply, reaches them with the WAN address it was sourced from and is routed back out that WAN's own table instead of following the main table out of the default WAN. + +Interface id 3 shows the effect of the permanent Tier 3 backstops. It is offline, so it has no Tier 1 (`iif`) rule, no Tier 2 (`fwmark 0x300/0x3f00 lookup 3`) rule and no Tier 4 source rule; those are installed only while the interface is up, so there is no `1003`, `2003` or `3064` entry. Its Tier 3 rule at priority `3003` is still present, because the backstops are installed for every configured WAN at service start and removed only at service stop. Traffic still marked `0x300` for the offline WAN therefore matches `3003` and is rejected as unreachable instead of falling through to the `main` table at priority 32766 and leaking out the default route. The backstop matches on the fwmark rather than on a device, which is why it can remain in place while the interface has no device bound. + The corresponding per-interface routing table (`ip route show table 1`) contains the WAN gateway and a copy of connected routes: ``` @@ -614,13 +658,23 @@ Shared helper library sourced by all mwan3 shell scripts. Provides: > [!NOTE] > **Shell scoping note:** Functions like `mwan3_id2mask` and `mwan3_count_one_bits` receive *variable names* as arguments (e.g., `mwan3_id2mask mmdefault MMX_MASK`) and use arithmetic expansion `$(($1))` to resolve them. This works in busybox ash (OpenWrt's default shell) because it uses dynamic scoping - local variables from the caller are visible in called functions. -### 6.3 `lib/mwan3/mwan3.sh` +### 6.3 `usr/share/ucode/mwan3/common.uc` + +Shared ucode library imported by the mwan3 ucode components, the ucode analogue of `common.sh`. Each shared ucode routine lives here so it is defined and tested in one place. Provides: + +- **Logging** (`log_open()`, `log_verbose()`, `log_msg()`): all mwan3 ucode logging goes through these so the sink, tag format, and verbose gate live in one place. Output goes to syslog via the `log` module, the same destination `logger(1)` reaches, so messages appear in `logread` regardless of how procd handles the process stderr. `log_open(ident)` opens the syslog connection with `LOG_PID` so the tag reads `ident[pid]`, matching the shell's `${SCRIPTNAME}[$$]` convention, under facility `daemon`. `log_verbose(enable)` sets the verbose gate; the caller reads `mwan3.globals.verbose_logging` and pushes the parsed bool. `log_msg(level, msg)` emits one message, dropping `debug` unless verbose exactly as `common.sh:LOG()` gates the debug facility, mapping mwan3's `warn`/`error` vocabulary and the syslog level names onto the tokens the `log` module accepts and passing the message as a `%s` argument so it is never treated as a format string. +- **`ucibool(val)`**: parses a UCI bool the way the shell's `config_get_bool` does (`1`/`on`/`true`/`yes`/`enabled` are true, everything else false), so every ucode consumer interprets a UCI bool identically to the shell. It reads no config itself, so it does not couple the module to uci. +- **`is_default_equivalent(route)`**: the width classifier for the bypass sweeps. Returns true when a route is a default in substance: no `dst` at all, or a prefix of `/2` or shorter for IPv4, `/3` or shorter for IPv6. The family is decided by `iptoarr` (inet_pton), so no address text is hand-parsed and a destination it cannot parse is not default-equivalent. Each route is judged alone, so half of a split pair is excluded on its own merits. The stricter literal-only predicate used by the route mirroring and `ROUTE_STATUS` paths is deliberately separate and is not widened. + +The module's public interface is declared as an export list (`export { log_open, log_verbose, log_msg, ucibool, is_default_equivalent }`) rather than inline on each function, which this ucode build does not accept. + +### 6.4 `lib/mwan3/mwan3.sh` The core engine. Contains all functions for managing nftables chains/sets/maps, ip rules, ip routes, policy creation, user rule classification, and status reporting. This is the largest file and the heart of the implementation. Sourced by init.d, hotplug, CLI, and rtmon scripts. See [Section 7](#7-function-reference) for detailed function reference. -### 6.4 `etc/init.d/mwan3` +### 6.5 `etc/init.d/mwan3` procd service script. Handles: @@ -644,6 +698,7 @@ mwan3_set_dynamic_sets() # populate dynamic sets mwan3_set_connected_sets() # populate connected sets mwan3_set_custom_sets() # populate custom sets mwan3_set_general_rules() # ip rule add (blackhole/unreachable) +mwan3_set_member_backstops() # ip rule add (per-WAN unreachable backstops) config_foreach mwan3_ifup interface "init" # trigger ifup hotplug per interface wait $hotplug_pids mwan3_set_general_nft() # populate hook chain rules @@ -655,7 +710,7 @@ mwan3_dnsmasq_hup() # HUP dnsmasq to re-populate nftset start rtmon_ipv4 + rtmon_ipv6 # route monitor daemons ``` -### 6.5 `etc/hotplug.d/iface/25-mwan3` +### 6.6 `etc/hotplug.d/iface/25-mwan3` Handles interface state change events from netifd. Triggered on `ifup`, `ifdown`, `connected`, and `disconnected` actions. @@ -675,17 +730,17 @@ There is no `fw4 reload` detection. `table inet mwan3` is unaffected by fw4 relo | Action | Operations | |---|---| -| `ifup` | Update peer track IP (if `track_gateway` enabled), create interface nft chain, create ip rules, set hotplug state, create routes, set general rules (if not init), rebuild policies (if online and not init). Signal tracker with USR2. | -| `ifdown` | Set offline state, delete map entries, delete ip rules, delete routes, delete interface nft chain. Signal tracker with USR1. Rebuild policies. | -| `connected` | Set online state, create interface nft chain, rebuild policies. | -| `disconnected` | Set offline state, rebuild policies. | +| `ifup` | Update peer track IP (if `track_gateway` enabled), create interface nft chain, create ip rules, create routes, set hotplug state. If not init: set general rules. If online and not init: rebuild policies (nft batch), flush worse-metric sticky sets, flush unreplied conntrack. Signal tracker with USR2. | +| `ifdown` | Set offline state. Rebuild policies + flush sticky sets (one nft batch). Delete ip rules, delete routes, delete interface nft chain. Signal tracker with USR1. Flush conntrack. | +| `connected` | Set online state. Create interface nft chain, create ip rules (no-op if already present), create routes (no-op if already present). Rebuild policies (nft batch). Flush worse-metric sticky sets. Flush unreplied conntrack. | +| `disconnected` | Set offline state. Rebuild policies + flush sticky sets (one nft batch). Flush conntrack. | All actions call `mwan3_flush_conntrack` at end. > [!NOTE] > **ifup conditional policy rebuild:** During init (`MWAN3_STARTUP=init`), the ifup action skips general rules and policy rebuild because the init sequence handles those after all interfaces are up. Route creation runs unconditionally on every ifup, including during init. During normal operation, policies are only rebuilt if the interface state is "online" (not for interfaces with `initial_state=offline`). -### 6.6 `usr/sbin/mwan3` (CLI) +### 6.7 `usr/sbin/mwan3` (CLI) User-facing command-line tool. Provides `start`/`stop`/`restart`/`ifup`/`ifdown` commands plus status reporting: `interfaces`, `policies`, `connected`, `rules`, `status` (all combined), and `internal` (detailed dump). @@ -693,7 +748,7 @@ The `use` command runs an arbitrary command bound to a specific interface using The `internal` command shows `nft list table inet mwan3` output instead of the old iptables dump. -### 6.7 `usr/sbin/mwan3rtmon` +### 6.8 `usr/sbin/mwan3rtmon` Route monitor daemon, reimplemented in **ucode**. Runs one instance per address family (ipv4/ipv6) as a procd service. Uses `ucode-mod-rtnl` for direct netlink access and `ucode-mod-uloop` for the event loop, eliminating all `ip` command fork+exec overhead from the original shell implementation. @@ -710,7 +765,7 @@ On startup, it performs an initial synchronization: dumps the current routing ta - **New route**: Adds CIDR networks to the connected set via `nft add element`, then replicates the route into active per-interface tables. Host routes (bare IPs without prefix length) are skipped as they are remote destinations. Schedules via the debouncer. - **Deleted route**: Schedules a debounced connected set rebuild, then removes the route from per-interface tables. -### 6.8 `usr/share/rpcd/ucode/mwan3` +### 6.9 `usr/share/rpcd/ucode/mwan3` ucode RPC service exposing ubus methods under the `mwan3` object. Used by LuCI for the web interface. @@ -727,9 +782,9 @@ Uses `nft -j` (JSON output mode) for reliable parsing. All set/chain queries are - **`mwan3.nftset_flush { set: "" }`**: Flushes all elements from the named set via `nft flush set inet mwan3 `. Returns `{}` on success or `{ error: "..." }` on failure. Used by the Flush button on the IP Sets status tab. - **`mwan3.nftset_reload { set: "" }`**: Flushes the set then re-adds all static entries from the UCI `config ipset` section (`list entry` values and the `loadfile` if configured). Returns `{}` on success or `{ error: "..." }` on failure. Used by the Reload button on the IP Sets status tab. - **`mwan3.nftset_resolve { set: "" }`**: Sends SIGHUP to dnsmasq to clear its cache, then queries each domain configured under the set's `list domain` option via the local dnsmasq instance (127.0.0.1). The DNS queries trigger dnsmasq's `nftset=` population mechanism as a side effect. Returns `{ resolved: N }` where N is the count of domains that resolved successfully. Only applicable for sets with `list domain` entries. Used by the Resolve button on the IP Sets status tab. -- **`mwan3.routing_health {}`**: Compares UCI configuration against live kernel state. Reads `iif_rule_base`, `fwmark_rule_base`, and `unreachable_rule_base` from UCI globals (with the same defaults and ordering constraint validation as `mwan3_init`) and derives `iface_max` dynamically from `mmx_mask`. Per interface, checks presence of the iif rule, fwmark lookup rule, and unreachable rule, and reports routing table default route presence. Reports stale ip rules across all three priority tiers. Returns a `rule_bases` object so the frontend can display configured priorities dynamically rather than assuming fixed offsets. +- **`mwan3.routing_health {}`**: Compares UCI configuration against live kernel state. Reads `iif_rule_base`, `fwmark_rule_base`, and `unreachable_rule_base` from UCI globals (with the same defaults and ordering constraint validation as `mwan3_init`), derives `iface_max` dynamically from `mmx_mask`, and derives the source base as `unreachable_rule_base + iface_max + 1`. Per interface, checks presence of the iif rule, fwmark lookup rule and unreachable rule, adds the source rule for interfaces of family `ipv4` (tested against the IPv4 rule set alone, since the source band is IPv4 only), and reports routing table default route presence. Reports stale ip rules across all four priority tiers. Returns a `rule_bases` object, the derived source base included, so the frontend can display configured priorities dynamically rather than assuming fixed offsets. -### 6.9 `Makefile` +### 6.10 `Makefile` Package build recipe. Key dependency changes: @@ -745,17 +800,19 @@ Package build recipe. Key dependency changes: | | `+ucode-mod-ubus` | | | `+ucode-mod-fs` | | | `+ucode-mod-socket` | +| | `+ucode-mod-log` | | | `+libnetfilter-conntrack` | `PKG_BUILD_DEPENDS` adds `libnetfilter_conntrack` and `libmnl` for the headers and libraries needed to compile `mwan3ct.c`. The runtime `+libnetfilter-conntrack` dependency provides the shared library that `mwan3ct` links against on the target. `mwan3ipcheck.c` is also compiled in the same build step. It links only against libc (`inet_pton`, `strtol`, standard string functions) and requires no additional build or runtime dependencies. -The ucode dependencies are required by the reimplemented `mwan3rtmon` route monitor daemon. `ucode-mod-socket` is required by `mwan3-diag` for address normalisation. +The ucode dependencies are required by the reimplemented `mwan3rtmon` route monitor daemon. `ucode-mod-socket` is required by `mwan3-diag` for address normalisation. `ucode-mod-log` provides the syslog binding used by the shared logging functions in `common.uc` (Section 6.3). Also installs: - `mwan3-skeleton.nft` to `$(1)/lib/mwan3/` +- `common.uc` to `$(1)/usr/share/ucode/mwan3/` (shared ucode library, see Section 6.3) - `mwan3-migrate-ipset-v4.sh` to `$(1)/lib/mwan3/` (one-shot migration helper, deleted from the router after postinst runs it) - `mwan3-remove-firewall-include` UCI defaults to `$(1)/etc/uci-defaults/` (removes legacy `firewall.mwan3_reload` UCI section from v3.x) - `mwan3-lb-test` to `$(1)/usr/sbin/` @@ -772,7 +829,7 @@ The `postinst` script: 5. Stops mwan3 a second time (by this point procd's auto-start has completed, so this stop removes auto-start's ip rules) 6. Starts mwan3 cleanly -### 6.10 `usr/sbin/mwan3track` +### 6.11 `usr/sbin/mwan3track` Interface health probe daemon. One procd service instance is launched per enabled mwan3 interface that has tracking IPs configured. Runs as a shell script; largely unchanged from the iptables version except for the addition of `track_gateway` and `check_quality` support. @@ -814,7 +871,7 @@ When `check_quality=1`, probes capture latency (ms) and packet loss (%) per IP. procd sends `SIGUSR1` (ifdown event) and `SIGUSR2` (ifup event) to trigger immediate state transitions without waiting for the next probe interval. -### 6.11 `usr/sbin/mwan3-lb-test` +### 6.12 `usr/sbin/mwan3-lb-test` Load balancing distribution verifier. @@ -839,7 +896,7 @@ See [§17.1](#171-mwan3-lb-test-load-balancing-distribution-verifier) for contex --- -### 6.12 `usr/sbin/mwan3-diag` +### 6.13 `usr/sbin/mwan3-diag` Network diagnostic report generator. @@ -857,7 +914,7 @@ See [§17.2](#172-mwan3-diag-network-diagnostic-report) for context. --- -### 6.13 `usr/sbin/mwan3ct` +### 6.14 `usr/sbin/mwan3ct` Targeted conntrack entry flush helper. @@ -885,7 +942,7 @@ Exit code 0 on success, including when no entries were matched. Errors are print --- -### 6.14 `lib/mwan3/mwan3-get-addr.uc` +### 6.15 `lib/mwan3/mwan3-get-addr.uc` Address lookup helper. @@ -898,32 +955,36 @@ Prints the address stripped of any prefix-length to stdout and exits 0, or exits --- -### 6.15 `lib/mwan3/mwan3-manage-rules.uc` +### 6.16 `lib/mwan3/mwan3-manage-rules.uc` Routing policy rule manager. -Manages ip rules (policy routing entries) via netlink (RTM_GETRULE, RTM_NEWRULE, RTM_DELRULE). Supports four modes: +Manages ip rules (policy routing entries) via netlink (RTM_GETRULE, RTM_NEWRULE, RTM_DELRULE, RTM_GETROUTE, RTM_GETADDR). Supports six modes: - **`check [prio2...]`**: returns a decimal bitmask where bit N is set if the Nth listed priority is absent from the live rule table. Used by mwan3.sh to decide which rules need to be created. - **`check-route `**: exits 0 if a default route for the given output device exists in the specified table, 1 otherwise. Used before populating per-interface tables to detect whether the route is already present. -- **`delete-iface `**: removes the iif lookup, fwmark lookup, and unreachable rules for a single interface (identified by its sequential index). Each rule is located by priority and verified by type before deletion; rules not present are silently skipped. +- **`delete-iface `**: removes the iif lookup, fwmark lookup and source lookup rules for a single interface (identified by its sequential index). Each rule is located by priority and verified by type before deletion, a source rule by carrying a source address at `src_prio`, so every address rule the interface holds goes in the one pass; rules not present are silently skipped. The unreachable backstop rule is not removed here; it is managed by `mwan3_set_member_backstops()` and persists until service stop. - **`add-general `**: adds the shared blackhole and unreachable rules common to all interfaces. Rules already present at those priorities are not re-added. +- **`add-backstop `**: adds one per-interface fwmark unreachable rule at the given priority, the Tier 3 backstop. A rule already present at that priority is not re-added. +- **`add-src `**: adds one source lookup rule per global-scope IPv4 address carried on the device, all at the given priority, the Tier 4 return rules. IPv4 only. Replaces `ip rule add/del` shell invocations that required fork-exec per operation and text parsing for presence checks. --- -### 6.16 `lib/mwan3/mwan3-list-routes.uc` +### 6.17 `lib/mwan3/mwan3-list-routes.uc` Route table lister. -Dumps routes from a given routing table via RTM_GETROUTE and prints destination prefixes to stdout, one per line. Accepts a numeric table ID or the special value `main`. Filters out: default routes (0.0.0.0/0, ::/0), host routes (prefix-length 32/128), link-local prefixes (fe80::/10, 169.254.0.0/16), IPv4 multicast (first octet >= 224), and duplicate destinations. +Dumps routes from a given routing table via RTM_GETROUTE and prints destination prefixes to stdout, one per line. Accepts a numeric table ID or the special value `main`. Filters out: default-equivalent routes, host routes (prefix-length 32/128), link-local prefixes (fe80::/10, 169.254.0.0/16), IPv4 multicast (first octet >= 224), and duplicate destinations. -Used by mwan3.sh to enumerate routes that need to be copied into per-interface routing tables. +A route is default-equivalent when it has no destination at all, or its prefix is `/2` or shorter for IPv4, `/3` or shorter for IPv6. That covers the literal defaults `0.0.0.0/0` and `::/0` and the covering forms some VPN clients install in their place, such as a `0.0.0.0/1` plus `128.0.0.0/1` pair or the IPv6 global-unicast prefix `2000::/3`; see the bypass set groups in Section 4. The test is `is_default_equivalent()` from `common.uc` (Section 7.2). + +Used by mwan3.sh to populate the `mwan3_connected_v4/v6` sets from the main table and the `mwan3_custom_v4/v6` sets from each table listed in `globals.rt_table_lookup`. --- -### 6.17 `lib/mwan3/mwan3-create-iface-route.uc` +### 6.18 `lib/mwan3/mwan3-create-iface-route.uc` Per-interface routing table populator. @@ -931,7 +992,7 @@ Copies routes from the main routing table (and any additional tables listed in t Replaces `ip route add table ` shell invocations that required fork-exec and text parsing of `ip route show` output. -### 6.18 `usr/bin/mwan3ipcheck` +### 6.19 `usr/bin/mwan3ipcheck` IP address and CIDR validation binary. Accepts a single argument and prints one of four classification strings to stdout, exiting 0 on success or 1 on failure. @@ -983,7 +1044,21 @@ The binary has no dependencies beyond libc (`inet_pton`, `strtol`, and standard | `get_uptime [out_var]` | Returns system uptime in seconds (integer). | | `get_online_time out_var iface` | Returns how long the interface has been online. | -### 7.2 Set Management Functions +### 7.2 common.uc Functions + +Functions in the shared ucode library `usr/share/ucode/mwan3/common.uc` (see [Section 6.3](#63-usrshareucodemwan3commonuc)), imported by the mwan3 ucode components. + +| Function | Purpose | +|---|---| +| `log_open(ident)` | Opens the syslog connection for the calling component with `LOG_PID` and facility `daemon`, so the tag reads `ident[pid]`, matching the shell components' `${SCRIPTNAME}[$$]` convention. | +| `log_verbose(enable)` | Sets the module-wide verbose gate. The caller reads `mwan3.globals.verbose_logging` (via `ucibool`) and pushes the parsed bool; `debug` messages are suppressed unless it is true, mirroring `common.sh:LOG()`. | +| `log_msg(level, msg)` | Emits one syslog message. `debug` is dropped unless verbose; every other level always logs. Maps mwan3's `warn`/`error` vocabulary and the syslog level names onto the tokens the `log` module accepts (an unknown level falls back to `notice`), and passes `msg` as a `%s` argument so it is never treated as a format string. | +| `ucibool(val)` | Parses a UCI bool the way the shell's `config_get_bool` does: `1`/`on`/`true`/`yes`/`enabled` are true, everything else false. Reads no config itself, so it does not couple the module to uci. | +| `is_default_equivalent(route)` | Returns true when a route is a default in substance: no `dst` at all, or a prefix of `/2` or shorter for IPv4, `/3` or shorter for IPv6. The family comes from `iptoarr` (inet_pton), so no address text is hand-parsed and an unparseable destination is not default-equivalent. Used by every producer that fills the connected and custom bypass sets. | + +### 7.3 mwan3.sh Functions + +#### 7.3.1 Set Management Functions | Function | Purpose | |---|---| @@ -998,14 +1073,15 @@ The binary has no dependencies beyond libc (`inet_pton`, `strtol`, and standard > [!NOTE] > **Why are connected functions self-contained?** `mwan3_set_connected_ipv4/ipv6` each manage their own batch because they may be called from contexts that are not already inside a larger batch (init.d, hotplug). `mwan3rtmon` does not call these shell functions; it has its own independent `populate_connected_set()` that operates via direct netlink calls and writes to the same sets using its own `nft_batch`. -### 7.3 General Rule Setup +#### 7.3.2 General Rule Setup | Function | Purpose | |---|---| | `mwan3_set_general_rules` | Adds `ip rule` entries for blackhole and unreachable marks (both IPv4 and IPv6) at priorities `MWAN3_FWMARK_RULE_BASE + MM_BLACKHOLE` and `MWAN3_FWMARK_RULE_BASE + MM_UNREACHABLE`. These are ip policy rules, not nftables rules. | +| `mwan3_set_member_backstops` | Installs one static unreachable `ip rule` per configured interface in the `MWAN3_UNREACHABLE_RULE_BASE + id` priority band. These rules are created at service start and reload and removed only at stop, so traffic carrying a member's mark is rejected rather than leaking to the main routing table when that member's fwmark rule is absent (interface down, disabled, or briefly mid-rebuild). Reserved last-resort marks are never enumerated here so `last_resort` behaviour is unaffected. | | `mwan3_set_general_nft` | Builds the per-mark OR setter chains via `mwan3_build_or_chains_nft()`, then populates all hook chain rules: IPv6 RA bypass (prerouting), vmap-dispatched connmark restore (if mark unset), jump to `mwan3_ifaces_in`, `fib daddr type local return` (prerouting, after ifaces_in), jumps to custom/connected/dynamic/rules chains, connmark save, and post-rules connected re-check. Idempotent: checks if rules already exist before adding. Uses a single batch for all operations. | -#### Rules added by `mwan3_set_general_nft()` +##### Rules added by `mwan3_set_general_nft()` For each of connected/custom/dynamic chains, adds mark-setting rules that match against the corresponding sets and apply `MMX_DEFAULT`. @@ -1022,7 +1098,7 @@ For the prerouting and output hook chains, adds (in order): See [§2 Connmark Operations](#connmark-operations) for the rationale and the kernel-limitation context. -### 7.4 Interface Management +#### 7.3.3 Interface Management | Function | Purpose | |---|---| @@ -1031,13 +1107,14 @@ See [§2 Connmark Operations](#connmark-operations) for the rationale and the ke | `mwan3_create_iface_nft iface device` | Creates (or flushes) `mwan3_iface_in_` chain. Adds rules matching on `iifname` and address family: source in connected/custom/dynamic → MMX_DEFAULT; otherwise → interface mark. Adds jump from `mwan3_ifaces_in` if not already present. Also installs a per-interface `mwan3_postrouting` SNAT rule for IPv6 interfaces when the `snat6` UCI option is set. Stale `mwan3_snat_`-tagged rules from a prior incarnation are removed first. See [§4](#router-originated-traffic-and-source-address-rewriting). | | `mwan3_rebuild_iface_nft iface` | Rebuilds a single interface's nft chain if the interface is enabled, the correct family is available, and the interface is currently up (verified via ubus). Used during `reload_service` to rebuild all interface chains within the atomic batch. Calls `mwan3_create_iface_nft()` after resolving the L3 device from netifd. | | `mwan3_delete_iface_nft iface` | Removes the jump rule from `mwan3_ifaces_in` (by handle lookup), removes any `mwan3_snat_`-tagged rules from `mwan3_postrouting` (comment-tag match), then flushes and deletes the interface chain. | -| `mwan3_delete_iface_map_entries iface` | Iterates all `mwan3_sticky_v[46]_*` sets in the `inet` family, finds sets whose name ends with `_` (this interface's id), and flushes them. Sets are flushed rather than deleted because rule chains may still reference the set name. | -| `mwan3_create_iface_rules iface device` | Adds `ip rule` entries: pref `id+MWAN3_IIF_RULE_BASE` (iif lookup), pref `id+MWAN3_FWMARK_RULE_BASE` (fwmark lookup), pref `id+MWAN3_UNREACHABLE_RULE_BASE` (fwmark unreachable). | -| `mwan3_delete_iface_rules iface` | Removes ip rules for this interface (both IPv4 and IPv6) by delegating to `mwan3-manage-rules.uc delete-iface`. The ucode script locates the iif lookup rule, fwmark lookup rule, and unreachable rule by priority offset and table ID, and deletes each via RTM_DELRULE. Handles rule-base changes and `mmx_mask` changes transparently. | +| `mwan3_delete_iface_map_entries iface` | Iterates all `mwan3_sticky_v[46]_*` sets in the `inet` family, finds sets whose name ends with `_` (this interface's id), and pushes a flush for each into the caller's open nft batch, so the flushes commit in the same transaction as the policy rebuild. The set enumeration reads committed kernel state and so stays outside the batch. Sets are flushed rather than deleted because rule chains may still reference the set name. Must be called within an open nft batch. | +| `mwan3_create_iface_rules iface device` | Adds `ip rule` entries: pref `id+MWAN3_IIF_RULE_BASE` (iif lookup) and pref `id+MWAN3_FWMARK_RULE_BASE` (fwmark lookup). For an IPv4 interface it also delegates to `mwan3-manage-rules.uc add-src` for the source lookup rules at pref `id+MWAN3_UNREACHABLE_RULE_BASE+MWAN3_INTERFACE_MAX+1`, one per global-scope IPv4 address on the device. The per-WAN unreachable backstop at `id+MWAN3_UNREACHABLE_RULE_BASE` is managed separately by `mwan3_set_member_backstops()` and is not installed here. | +| `mwan3_rebuild_iface_rules iface` | Rebuilds a single interface's ip rules and routing table if the interface is enabled, its family is available, and it is currently up with an L3 device (verified via ubus). Called from `reload_service` via `config_foreach`, after the nft batch has committed, because ip rules and routes are kernel state that nftables does not own. Calls `mwan3_create_iface_rules()`, which deletes the interface's existing rules before adding them, then flushes and repopulates the routing table via `mwan3_delete_iface_route()` and `mwan3_create_iface_route()`. Unlike `mwan3_rebuild_iface_nft()` it does not consult the hotplug state file, so an interface that is up at netifd but tracked down keeps its routing in place. | +| `mwan3_delete_iface_rules iface` | Removes the iif lookup, fwmark lookup and source lookup ip rules for this interface (both IPv4 and IPv6) by delegating to `mwan3-manage-rules.uc delete-iface`. The ucode script locates each rule by priority offset and table ID and deletes it via RTM_DELRULE. Handles rule-base changes and `mmx_mask` changes transparently. The unreachable backstop at `id+MWAN3_UNREACHABLE_RULE_BASE` is not deleted here; it persists until service stop. | | `mwan3_create_iface_route iface` | Copies routes from the main routing table (and any `rt_table_lookup` extra tables) into the per-interface routing table by invoking `mwan3-create-iface-route.uc`. The ucode script reads source routes via rtnl, skips routes already present in the target table, and writes each qualifying route with RTM_NEWROUTE. Passes the `source_routing` UCI flag to control whether source addresses are copied. | | `mwan3_delete_iface_route iface` | Flushes the per-interface routing table. *Unchanged.* | -#### Handle-Based Rule Deletion +##### Handle-Based Rule Deletion nftables doesn't support deleting rules by match criteria (like iptables `-D chain match...`). Instead, `mwan3_delete_iface_nft()` uses: @@ -1049,29 +1126,30 @@ $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle" The `-a` flag shows rule handles in comments, which can then be used for targeted deletion. -### 7.5 Policy & Load Balancing +#### 7.3.4 Policy & Load Balancing | Function | Purpose | |---|---| -| `mwan3_set_policy member_config` | Callback per policy member. Tracks lowest metric and accumulates online members as `iface:id:weight` tuples into `$policy_members`. Tracks offline devices into `$policy_offline_devices`. Uses caller's variables (dynamic scoping). | -| `mwan3_create_policies_nft policy` | Creates/flushes the `mwan3_policy_` chain. Iterates members via `mwan3_set_policy`, then builds the chain: single member gets a direct mark-set rule; multiple members get a `numgen` rule; offline devices get out-device fallback rules; last-resort rule (unreachable/blackhole/default) is appended. | +| `mwan3_set_policy member_config` | Callback per policy member. Tracks lowest metric per address family and accumulates online members as `iface:id:weight` tuples into `$policy_members_v4` and `$policy_members_v6`. Uses caller's variables (dynamic scoping). | +| `mwan3_create_policies_nft policy` | Creates/flushes the `mwan3_policy_` chain. Iterates members via `mwan3_set_policy`, then builds the chain: single online member gets a direct mark-set rule; multiple online members get a `numgen` rule; last-resort rule (unreachable/blackhole/default) is appended. | | `mwan3_set_policies_nft` | Before creating policy chains, enumerates all existing `mwan3_policy_*` chains in `inet mwan3` and deletes any whose name is not in the current UCI policy config (orphaned chain sweep). Then iterates all policy configs and calls `mwan3_create_policies_nft` for each. | -### 7.6 Sticky Routing +#### 7.3.5 Sticky Routing | Function | Purpose | |---|---| | `mwan3_get_policy_members_for_family policy family` | Iterates the members of a policy config via `config_list_foreach`. For each member whose interface matches the requested family, resolves the interface id and computes its mark, and accumulates `:` tuples into `$_policy_member_marks`. Used by the sticky path in `mwan3_set_user_nft_rule()` to enumerate per-member sets. | +| `mwan3_flush_worse_metric_sticky iface` | On recovery of a higher-priority interface, flushes the per-rule sticky sets of lower-metric same-family members of each policy the recovered interface participates in, scoped to the rules that use that policy. Set names embed the rule name and member id, so a member that appears in a separate load-balancing policy with equal-ranked peers keeps those pins untouched. Called after the policy rebuild on the `connected` and `ifup` (online) hotplug arms. | -### 7.7 User Rules +#### 7.3.6 User Rules | Function | Purpose | |---|---| | `mwan3_set_user_nft_rule rule ipv` | Builds and adds a single nft rule to `mwan3_rules`. Translates UCI config options (proto, src_ip, dest_ip, src_port, dest_port, src_iface, ipset, ipset_src) into nft match expressions. For sticky rules, calls `mwan3_get_policy_members_for_family()` and builds a `mwan3_rule_` chain with per-member address sets and OR-immediate restore/save (see §8). Handles logging rules. Pre-creates missing nft sets (e.g., dnsmasq nftsets that haven't started yet) to prevent batch failures. When `family` is explicitly `ipv4` or `ipv6` but none of `src_ip`, `dest_ip`, `ipset`, or `ipset_src` are set, prepends `meta nfproto ipv4/ipv6` to the nft match to prevent a bare rule from matching both address families. Also translates `proto icmp` to `meta l4proto ipv6-icmp` when `family` is `ipv6` (in an inet table, proto 1 is ICMPv4 only). `src_ip` and `dest_ip` are validated via `mwan3ipcheck`; a result of `"invalid"` or `"mixed"` causes the rule to be skipped with a warning log. When a comma is present in `src_ip` or `dest_ip`, the value is wrapped in nft anonymous set syntax (`{ addr1, addr2 }`) for the generated match expression. | -| `mwan3_set_user_rules` | Flushes `mwan3_rules` chain, then iterates all rule configs for both ipv4 and ipv6, calling `mwan3_set_user_nft_rule` for each. Uses a single batch. | +| `mwan3_set_user_rules` | Flushes `mwan3_rules` chain, then iterates all rule configs in config order, calling `mwan3_set_user_nft_rule`for both IPv4 and IPv6. Uses a single batch. | | `mwan3_set_user_iface_rules iface device` | Called on ifup to check if the rules chain needs rebuilding (if this interface is a `src_iface` in any rule). | -#### UCI-to-nft Match Translation +##### UCI-to-nft Match Translation | UCI Option | nft Expression | |---|---| @@ -1088,22 +1166,24 @@ The `-a` flag shows rule handles in comments, which can then be used for targete | `use_policy balanced` | `jump mwan3_policy_balanced` | | `use_policy default` | `meta mark set ... \| MMX_DEFAULT` | -#### Missing nft Set Pre-creation +##### Missing nft Set Pre-creation When a user rule references an `ipset` (destination nft set) or `ipset_src` (source nft set) that doesn't exist yet in `table inet mwan3`, `nft -f` would roll back the entire batch atomically. To prevent this, `mwan3_set_user_nft_rule()` pre-creates any missing referenced sets with the appropriate type. The same logic applies to both `ipset` and `ipset_src`. -### 7.8 User-defined nft Set Management +A set is pre-created at most once per render. Kernel queries inside an open batch see pre-batch state, so the renderer carries its own record of the sets it has already added to the batch, as `name=addrtype` tokens reset at the start of each render. The first rule to reference a missing set settles its address type for the whole render; a later rule referencing the same name from the other family is skipped, silently when it is a `family=any` rule and with a warning when its `family` is explicit. Without the record, two rules of different families referencing one missing set each added it with a conflicting type, nft rejected the batch, and the router was left with no user rules at all. + +#### 7.3.7 User-defined nft Set Management | Function | Purpose | |---|---| | `mwan3_render_config_ipsets` | Iterates all `config ipset` UCI sections and calls `_mwan3_render_one_ipset` for each. Creates user-defined sets in `table inet mwan3`. Called from `start_service` and `reload_service`. | | `_mwan3_nft_time_to_sec str` | Converts an nft human-unit timeout string (e.g. `1h`, `5m`, `300s`) to an integer number of seconds. Recognised suffixes: `d` (days), `h` (hours), `m` (minutes), `s` (seconds, default if no suffix). Used by `_mwan3_ipset_needs_delete` to compare the live kernel timeout against the desired spec in a common unit. | | `_mwan3_ipset_needs_delete name type counter timeout maxelem` | Returns 0 (true) if the named set exists in `table inet mwan3` AND its flags (type, counter, timeout, size) differ from the desired spec; returns 1 if the set is absent or all flags match. Called by `_mwan3_render_one_ipset` during reload to decide whether to queue a `delete set` before recreating the set with new flags. Timeout values are compared in seconds after converting nft's human-unit display (e.g. `1h`, `5m`) via `_mwan3_nft_time_to_sec`. | -| `_mwan3_render_one_ipset name` | Creates one user-defined nft set. Builds a `type ipv4_addr/ipv6_addr; flags interval; auto-merge;` declaration, optionally appending `counter;` (standalone statement, NOT a flag keyword) and `timeout Ns;`. On the start path, unconditionally deletes any existing set before creating. On the reload path, calls `_mwan3_ipset_needs_delete` and only queues a `delete set` when flags differ; otherwise `add set` is idempotent and dnsmasq-populated elements survive. Sets `MWAN3_NEED_DNSMASQ_HUP` if a domain-populated set is deleted. Adds inline `list entry` elements and `loadfile` contents. | +| `_mwan3_render_one_ipset name` | Creates one user-defined nft set. Builds a `type ipv4_addr/ipv6_addr; flags interval;` declaration; appends `auto-merge;` only when no timeout is configured, since auto-merge coalesces contiguous addresses into merged ranges with a single expiry, silently discarding per-element TTLs in DNS-populated sets. Optionally appends `counter;` (standalone statement, NOT a flag keyword) and `timeout Ns;`. On the start path, unconditionally deletes any existing set before creating. On the reload path, calls `_mwan3_ipset_needs_delete` and only queues a `delete set` when flags differ; otherwise `add set` is idempotent and dnsmasq-populated elements survive. Sets `MWAN3_NEED_DNSMASQ_HUP` if a domain-populated set is deleted. Adds inline `list entry` elements and `loadfile` contents. | | `mwan3_write_dnsmasq_fragments` | For each `config ipset` with `list domain` entries, writes a dnsmasq confdir fragment enabling `nftset=/domain/FAMILY#inet#mwan3#setname`. Compares against previously written fragments; restarts dnsmasq only when content changes. Silently does nothing if no domain sets are configured. | | `mwan3_cleanup_orphaned_ipsets` | Queries `nft list table inet mwan3` for user-defined sets (those not prefixed `mwan3_`), compares against configured set names, and pushes `delete set` for any orphans. Called in `reload_service` after `mwan3_render_config_ipsets`. Prevents stale sets accumulating when a set is removed from UCI config. | -### 7.9 Status Reporting +#### 7.3.8 Status Reporting | Function | Purpose | |---|---| @@ -1115,7 +1195,7 @@ When a user rule references an `ipset` (destination nft set) or `ipset_src` (sou | `mwan3_report_rules_v4/v6` | Parses `nft list chain inet mwan3 mwan3_rules`. | | `mwan3_mark_to_name mark` | Resolves a numeric mark value to an interface name (or "default"/"blackhole"/"unreachable"). | -### 7.10 Lifecycle & Hotplug +#### 7.3.9 Lifecycle & Hotplug | Function | Purpose | |---|---| @@ -1124,7 +1204,7 @@ When a user rule references an `ipset` (destination nft set) or `ipset_src` (sou | `mwan3_interface_shutdown iface` | Calls hotplug shutdown then cleans track state files. | | `mwan3_set_iface_hotplug_state iface state` | Writes state (`online`/`offline`) to status file. | | `mwan3_get_iface_hotplug_state iface` | Reads state from status file (defaults to `offline`). | -| `mwan3_flush_conntrack iface action` | Two-path conntrack flush. First, iterates the `flush_conntrack` UCI list for this interface: for each configured action that matches the current hotplug action, writes `f` to `$CONNTRACK_FILE` to flush the entire global conntrack table. Second, on `ifdown`, uses `conntrack -D --mark MARK/MMX_MASK` to selectively delete only the conntrack entries for this interface's fwmark. Both paths run; the global flush path runs first. | +| `mwan3_flush_conntrack iface action` | Two-path conntrack flush. First, iterates the `flush_conntrack` UCI list for this interface: for each configured action that matches the current hotplug action, writes `f` to `$CONNTRACK_FILE` to flush the entire global conntrack table. Second, on `ifdown` and `disconnected`, uses `mwan3ct flush --mark MARK/MMX_MASK` to selectively delete only the conntrack entries for this interface's fwmark. Both paths run; the global flush path runs first. | | `mwan3_flush_marked_conntrack` | Flushes all conntrack entries that have any `MMX_MASK` bit set, iterating the full mwan3 id-space (IDs 1..`MWAN3_INTERFACE_MAX` plus default/blackhole/unreachable marks). Called from `reload_service` so that live flows re-enter the classification chains and are re-evaluated against new rules rather than staying pinned to a previously saved ct mark. | | `mwan3_update_peer_track_ip iface` | If `track_gateway` is enabled, queries `ifstatus` for the point-to-point peer address and writes it to `$MWAN3TRACK_STATUS_DIR//GATEWAY`. | | `mwan3_track_clean iface` | Removes track status directory for the interface. | @@ -1132,6 +1212,70 @@ When a user rule references an `ipset` (destination nft set) or `ipset_src` (sou | `mwan3_flush_stale_conntrack` | Flushes conntrack entries with no mwan3 mark (`0x0/MMX_MASK`) after mwan3 restart. New connections arriving during the brief startup window before iface_in chains exist get ct mark=0; this clears them so they re-establish correctly. Called from `start_service`. | | `mwan3_flush_unreplied_conntrack` | Flushes all mwan3-marked conntrack entries that have never received a reply (`IPS_SEEN_REPLY` not set). Called after `mwan3_set_policies_nft` on the `connected` action and on `ifup` when `status=online`. Breaks the cycle where a stale mark from a previous policy state pins traffic to the wrong WAN: because no reply arrives the application keeps retrying, refreshing the conntrack timeout indefinitely. Flushing only UNREPLIED entries leaves established connections untouched. | +### 7.4 mwan3rtmon Functions + +`usr/sbin/mwan3rtmon` (see [Section 6.8](#68-usrsbinmwan3rtmon)) is the ucode route-monitor daemon; one instance runs per address family (`ipv4` and `ipv6`). It mirrors kernel routing changes into mwan3's connected and custom nft sets and into the per-interface routing tables, and maintains each tracked interface's `ROUTE_STATUS` file. Shared logging (`log_open`, `log_verbose`, `log_msg`), `ucibool` and `is_default_equivalent` are imported from `common.uc` (Section 7.2). + +| Function | Purpose | +|---|---| +| `main()` | Entry point. Parses the address family from `ARGV[0]` (`ipv4`/`ipv6`), opens syslog with `log_open`, loads config, connects to ubus, and binds the rtnl route listener before running the initial dumps so route changes during the dump phase are queued rather than lost to a subscribe-vs-dump race. Runs the initial custom-set, connected-set, per-interface-route, and route-state population, installs SIGTERM/SIGINT/SIGHUP handlers (SIGHUP reloads config and re-sweeps), arms a 20-second defensive route-state re-sweep (the nested `schedule_periodic_route_sweep()`), then enters the uloop event loop. One process runs per family. | +| `log_msg(level, msg)` | Local logging wrapper that prefixes the `[family]` tag and delegates to `common.uc`'s `log_msg` (imported as `mlog`). | +| `load_config()` | Loads the `mwan3` UCI config: sets the verbose gate via `log_verbose(ucibool(...))`, reads `mmx_mask` and `source_routing`, builds `extra_table_set` from `rt_table_lookup`, and builds `iface_list` (name, family, sequential table id, enabled, has_tracking) in UCI declaration order. Called at startup and on SIGHUP. | +| `build_dev_table_map()` | From the ubus `network.interface` dump, maps each enabled same-family interface's `l3_device` to its table id in `dev_table_map`. | +| `refresh_active_chains()` | Caches the set of existing `inet mwan3` chains from `nft -j list chains inet` into `active_chains`. | +| `is_iface_nft_active(ifname)` | Returns whether `mwan3_iface_in_` is present in the cached chain set. | +| `get_active_tids()` | Returns a `{tid: name}` map of same-family interfaces whose per-interface nft chain is currently active. | +| `get_track_status(ifname)` | Reads the tracker's `PID` and `STARTED` status files and returns `disabled`, `down`, `paused`, or `active`, mirroring the shell tracker-status logic. | +| `nft_exec(cmd)` | Runs a single `nft` command, logging a warning with the command and its output on non-zero exit. | +| `nft_batch(cmds)` | Writes a list of commands to a per-family batch file, `/var/run/mwan3/mwan3rtmon_nft_batch.`, and applies them atomically with `nft -f`, logging on failure. One instance runs per address family and each is single-threaded, so the family suffix alone keeps the two instances' files apart without locking; the file is removed on every exit path after creation. | +| `build_route_for_table(route, tid)` | Builds a route object targeting table `tid`, copying the relevant `ROUTE_FIELDS`, dropping `tos == 0`, and including `src` only when `source_routing` is enabled. | +| `apply_route_to_table(route, tid, is_new)` | Adds (`RTM_NEWROUTE`, create+replace) or deletes (`RTM_DELROUTE`) the route in table `tid` via rtnl, logging add failures. | +| `is_cidr_route(route)` | Returns true when the route's `dst` prefix length is shorter than the family maximum (32/128), i.e. a network route rather than a host route. | +| `is_default_route(route)` | Returns true for a null `dst`, `0.0.0.0/0`, or `::/0`. Route mirroring and the `ROUTE_STATUS` check rely on this narrow meaning, so it is deliberately left strict. The bypass sweeps use the wider `is_default_equivalent` from `common.uc` instead. | +| `is_linklocal_route(route)` | Returns true for a `dst` under `fe80::/` or `169.254.`. | +| `table_has_default(routes, tid, dev)` | Returns true if the route dump holds a default route in table `tid` via device `dev`. This is the datapath predicate behind `ROUTE_STATUS`: the interface's fwmark ip rule looks up exactly this table, so a table with no default black-holes the traffic marked to it even when the main table still has a default via the same device. | +| `tracked_interfaces()` | Returns `[{name, tid, dev}]` for the enabled, tracked, same-family interfaces currently bound to an l3 device (device from the ubus dump). Interfaces with no device yet are skipped and re-evaluated on a later route change. | +| `write_route_state(ifname, state)` | Atomically writes the interface's `ROUTE_STATUS` file (temp file + rename in the same tmpfs directory) only when the value changes, and logs the transition (`down` at warn, the return at notice). | +| `refresh_route_states()` | Recomputes every responsible interface's route state from one full route dump and persists it via `write_route_state`. Called at startup, on SIGHUP, from the periodic sweep, and debounced from the route-event handler. | +| `arm_route_state_refresh()` | Arms or reschedules the debounce timer that collapses a burst of route events into a single `refresh_route_states` sweep. | +| `get_all_routes()` | Dumps routes for this family and filters to the main table plus any `rt_table_lookup` extra tables. | +| `route_still_exists(route)` | Re-dumps and checks whether a route still exists in the main table; used to verify delete events before acting on them. | +| `populate_connected_set()` | Flushes and repopulates `mwan3_connected_v4/v6` from CIDR, non-default-equivalent, non-link-local routes (plus `224.0.0.0/3` for IPv4), gated by a sorted-element fingerprint so an unchanged set skips the nft write. | +| `populate_iface_routes()` | Copies main and extra-table routes into the active per-interface tables: a device-matched route goes to that interface's table, other network routes broadcast to all active tables. | +| `repopulate_custom_sets()` | Rebuilds `mwan3_custom_v4/v6` from the `rt_table_lookup` tables, excluding default-equivalent and link-local routes; used at startup and on SIGHUP reload. | +| `handle_custom_set_event(route, is_new)` | Mirrors a single `rt_table_lookup` route add or delete into `mwan3_custom_v4/v6` so those destinations receive `MMX_DEFAULT` and bypass WAN policy selection; default-equivalent and link-local routes are skipped. | +| `handle_route_event(event)` | The rtnl listener callback for `RTM_NEWROUTE`/`RTM_DELROUTE`: mirrors `rt_table_lookup` routes into the custom set, debounces the connected-set rebuild, arms the route-state refresh, and updates the per-interface tables, verifying delete events and gating broadcast routes on tracker status. | + +### 7.5 mwan3track Functions + +`usr/sbin/mwan3track` (see [Section 6.11](#611-usrsbinmwan3track)) is the per-interface tracker; one shell process runs per tracked interface. It probes the interface's `track_ip` targets on an interval, maintains a hysteresis score, writes per-target and per-interface status files under `$MWAN3TRACK_STATUS_DIR//`, and fires `connected`/`disconnected` (and the transitional `connecting`/`disconnecting`) hotplug events as the score crosses its thresholds. Signals drive its lifecycle: `USR1` is ifdown, `USR2` is ifup, `HUP` is config reload, `TERM` is stop. + +| Function | Purpose | +|---|---| +| `stop_subprocs` | Kills the tracker's background jobs (the running probe and sleep subshells). Used by the signal handlers to break out of the current round. | +| `WRAP cmd...` | Runs the probe command `cmd` under `LD_PRELOAD` of the mwan3 sockopt wrapper, exporting `FAMILY`, `DEVICE`, `SRCIP`, and `FWMARK` so the probe leaves via the interface's own address and is marked for its own table. | +| `clean_up` | `TERM` handler: logs the stop, kills subprocesses, and exits. | +| `if_down` | `USR1` handler: sets `IFDOWN_EVENT` and stops subprocesses so the loop registers the ifdown. | +| `if_up` | `USR2` handler: sets `IFUP_EVENT`, marks the tracker started, and stops subprocesses. | +| `config_reload` | `HUP` handler: sets `RELOAD_EVENT` and stops subprocesses so the loop reloads config. | +| `ping_test_host` | Echoes the family's loopback address (`127.0.0.1` or `::1`), used to test whether a ping variant works. | +| `get_ping_command` | Selects the usable ping binary/variant (`ping -4/-6`, `ping6`, or `/bin/ping`) and echoes it; returns non-zero if none is available. | +| `validate_track_method method` | Verifies the tool for the configured `track_method` (`ping`/`arping`/`httping`/`nping-*`/`nslookup`) is installed, logging a warning and returning non-zero if not. | +| `validate_wrap` | Ensures the `libwrap_mwan3_sockopt` shared library exists; logs an error and exits if it is missing. | +| `disconnected [suppress]` | Marks the interface offline: writes `STATUS`/`OFFLINE`/`ONLINE`, zeroes the score, and (unless `suppress` is `1`, as on first connect) fires the `disconnected` hotplug event when the previous status was online or disconnecting. | +| `connected [firstconnect]` | Marks the interface online: writes `STATUS`/`ONLINE`/`OFFLINE`, resets the per-round counters, and fires the `connected` hotplug event (passing `FIRSTCONNECT`). | +| `disconnecting` | Enters the transitional `disconnecting` state once and fires the `disconnecting` hotplug event, an early warning before the interface is declared offline. | +| `connecting` | Enters the transitional `connecting` state once and fires the `connecting` hotplug event, an early signal before the interface is declared online. | +| `disabled` | Marks the interface `disabled` and calls `stopped`. | +| `firstconnect` | Resolves the true interface, L3 device, and source IP, determines the initial status (`initial_state`, default online), marks the tracker started, then calls `connected` or `disconnected` accordingly. Called at startup and after ifup. | +| `stopped` | Sets `STARTED=0` and writes the `STARTED` status file; the tracker then idles until a signal. | +| `started` | Sets `STARTED=1` and writes the `STARTED` status file. | +| `update_status track_ip state [latency loss]` | Writes the per-target `TRACK_` status file (and, when supplied, `LATENCY_`/`LOSS_`) for one probe target. | +| `load_tracking_config` | Reads all per-interface tracking UCI options (family, track_method, reliability, count, timeout, interval, up/down thresholds, size, max_ttl, failure/recovery intervals, check_quality and its latency/loss thresholds), clamping `check_quality` off for track methods that produce no latency/loss samples. Called at startup and on `HUP` reload. | +| `main iface` | Entry point and probe loop. Takes the per-interface lock and writes `PID`, installs the signal traps, loads config and the probe-target list, then loops: each round it reads `ROUTE_STATUS` (a `down` written by `mwan3rtmon` counts as a failed round with no probing), probes each target with `WRAP`, adjusts the hysteresis score, and calls the state transitions; between rounds it services pending ifdown/ifup/reload events. | + +The nested helpers `mwan3_load_track_ips` and `mwan3_list_track_ips`, defined inside `main()`, assemble the probe-target list (static `track_ip` entries plus the discovered gateway when `track_gateway` is set) and prune stale per-target status files; they are not listed separately. + --- ## 8. Load Balancing with numgen @@ -1162,8 +1306,7 @@ nft add rule inet mwan3 mwan3_policy_balanced \ 4. Calculate total weight = sum of all member weights 5. If single member: direct `meta mark set` (no numgen overhead) 6. If multiple members: build numgen map entries with ranges proportional to weight -7. Append offline device fallback rules (only if no online members) -8. Append last-resort rule (unreachable/blackhole/default) +7. Append last-resort rule (unreachable/blackhole/default) > [!NOTE] > **Difference from iptables version:** The iptables version used probabilistic matching (`--probability`) which is statistically correct over many packets but can have short-term imbalance. The nftables `numgen inc` counter gives perfectly deterministic round-robin distribution at the configured weights. @@ -1207,6 +1350,10 @@ chain mwan3_rule_https { } ``` +The chain is built once, complete for every family whose traffic can reach it. A rule with an explicit `family`, or with a family-anchoring match element (`src_ip`, `dest_ip`, `ipset`, `ipset_src`), reaches the chain in one address family only and gets that family's arms. A `family=any` rule with no such element renders as a single family-agnostic line in `mwan3_rules`, so traffic of both families reaches its chain; the IPv4 pass therefore builds both arms, `ip saddr` lookups and updates against the `mwan3_sticky_v4_*` sets and `ip6 saddr` against the `mwan3_sticky_v6_*` sets. `proto icmp` is the exception that renders a line on both passes, because ICMP and ICMPv6 are distinct L4 protocols; the IPv6 pass then points its own line at the chain the IPv4 pass has already built rather than appending to it. + +The layout is load-bearing: every lookup precedes the single fall-through to the policy, and the update rules follow it. A pinned source must OR its member mark in before the fall-through can hand the packet to the policy. A lookup placed after the fall-through would let the policy re-mark the flow first and the late lookup would then OR the old pin into the new mark, producing a value that matches no interface. + ### Flow for a Sticky Rule 1. Packet arrives at `mwan3_rules` chain @@ -1215,6 +1362,12 @@ chain mwan3_rule_https { 4. **New source**: no set match, mark stays 0. Falls through to the policy chain which assigns a mark. The matching update rule then adds saddr to the corresponding member's set with the configured timeout. 5. After timeout seconds of inactivity the set entry expires and the source is re-evaluated at next connection. +### Set Lifecycle + +A sticky entry normally persists until its per-entry timeout, but two failover events also flush sticky sets directly, so a pinned source can be re-evaluated before its timeout expires. When a member fails (a soft `disconnected` failure with the link still up) or goes down (`ifdown`), that member's own sticky sets are flushed by `mwan3_delete_iface_map_entries` in the same nft transaction that drops it from the policy, so a source pinned to the failed member re-picks the survivor on its next new connection rather than staying nailed to a member that is gone. When a higher-priority member recovers, `mwan3_flush_worse_metric_sticky` flushes the per-rule sticky sets of the lower-metric same-family members of each policy the recovered member participates in, scoped to the rules that use that policy (the snap-back), so new flows return to the recovered member; an equal-ranked load-balancing policy clears nothing, because no member outranks another. + +In every case only the sticky sets are touched, and only new flows re-pick as a result. Established connections are not moved, because they are pinned by their saved conntrack mark rather than by the sticky sets (see Sections 10.3 and 10.4). + --- ## 10. Service Lifecycle and Conntrack Management @@ -1227,11 +1380,12 @@ chain mwan3_rule_https { +--> mwan3_init() compute masks +--> mwan3_ensure_nft_framework() recreate 6 internal sets, ensure chains +--> mwan3_render_config_ipsets() create user-defined sets from config ipset - +--> mwan3_write_dnsmasq_fragments() write nftset confdir fragments; restart dnsmasq if changed + +--> mwan3_write_dnsmasq_fragments() write nftset fragments; restart if changed +--> start_tracker per interface launch mwan3track +--> mwan3_update_iface_to_table() build iface->table mapping +--> mwan3_set_dynamic/connected/custom sets +--> mwan3_set_general_rules() ip rule add blackhole/unreachable + +--> mwan3_set_member_backstops() per-WAN unreachable backstop ip rules +--> mwan3_ifup per interface trigger hotplug (creates iface chains + ip rules) +--> wait for hotplug completion +--> mwan3_set_general_nft() populate hook chain rules @@ -1247,7 +1401,11 @@ chain mwan3_rule_https { ``` /etc/init.d/mwan3 reload - +--> mwan3_nft_reload_start() open batch + write preamble (flush/delete dynamic state) + +--> [table inet mwan3 absent] stop + start, then return + (the skeleton load that creates the + table is skipped when every interface + is disabled) + +--> mwan3_nft_reload_start() open batch, flush/delete dynamic state +--> mwan3_ensure_nft_framework() recreate 6 internal sets +--> mwan3_render_config_ipsets() add user-defined sets (skips delete inside batch) +--> mwan3_cleanup_orphaned_ipsets() delete user sets not in current config @@ -1259,41 +1417,48 @@ chain mwan3_rule_https { +--> mwan3_nft_reload_commit() commit all as single atomic kernel transaction +--> mwan3_update_iface_to_table() (outside nft) +--> mwan3_set_general_rules() (outside nft) + +--> mwan3_set_member_backstops() (outside nft) +--> mwan3_write_dnsmasq_fragments() restart dnsmasq only if fragment content changed - +--> [if MWAN3_NEED_DNSMASQ_HUP=1] mwan3_dnsmasq_hup() repopulate domain sets lost on flag change + +--> [if MWAN3_NEED_DNSMASQ_HUP=1] mwan3_dnsmasq_hup() repopulates + domain sets lost on a flag change +--> [tracker count check] if mismatch: stop + start (procd cannot add/remove service instances in reload_service) ``` User-defined sets (config ipset) and sticky sets (`mwan3_sticky_*`) survive the reload intact - they are never in the preamble delete path. -### 10.3 Interface Up (hotplug) +### 10.3 Interface Up and Tracker Recovery (hotplug) ``` netifd signals ifup for $INTERFACE +--> 25-mwan3 hotplug script - +--> mwan3_update_peer_track_ip() write gateway IP (if track_gateway) - +--> mwan3_create_iface_nft() create/flush chain, add rules - +--> mwan3_create_iface_rules() ip rule add (iif, fwmark, unreachable) - +--> mwan3_create_iface_route() copy routes to per-iface table + +--> mwan3_update_peer_track_ip() write gateway IP (track_gateway) + +--> mwan3_create_iface_nft() create/flush chain, add rules + +--> mwan3_create_iface_rules() ip rule add (iif, fwmark, source) + +--> mwan3_create_iface_route() copy routes to per-iface table +--> mwan3_set_iface_hotplug_state "online/offline" +--> [if not init startup:] - | +--> mwan3_set_general_rules() ensure ip rules exist - | +--> [if online:] mwan3_set_policies_nft() rebuild policy chains - | mwan3_flush_unreplied_conntrack() flush UNREPLIED entries + | +--> mwan3_set_general_rules() ensure ip rules exist + | +--> [if online:] + | +--> [nft batch:] mwan3_set_policies_nft() + | +--> mwan3_flush_worse_metric_sticky() + | +--> mwan3_flush_unreplied_conntrack() +--> procd_send_signal track_$INTERFACE USR2 ``` ``` -netifd signals connected for $INTERFACE +connected fires for $INTERFACE (link-layer negotiation, or tracker recovery) +--> 25-mwan3 hotplug script +--> mwan3_set_iface_hotplug_state "online" - +--> mwan3_create_iface_nft() flush chain, update rules - +--> mwan3_set_policies_nft() rebuild policy chains - +--> mwan3_flush_unreplied_conntrack() flush UNREPLIED entries + +--> mwan3_create_iface_nft() flush chain, update rules + +--> mwan3_create_iface_rules() recreate ip rules (idempotent) + +--> mwan3_create_iface_route() recreate routes (idempotent) + +--> [nft batch:] mwan3_set_policies_nft() + +--> mwan3_flush_worse_metric_sticky() + +--> mwan3_flush_unreplied_conntrack() ``` -The `connected` event fires when an interface completes its link-layer negotiation (e.g., PPPoE authentication) after the `ifup` routing state is already in place. It skips rule and route creation (those are done on `ifup`) and goes straight to policy rebuild and the unreplied flush. +The `connected` action fires on two distinct events: link-layer negotiation completion (PPPoE authentication, for example) after `ifup` routing state is already in place, and tracker recovery, when mwan3track judges a previously-failed interface to be working again without a physical reconnect. On a tracker recovery the interface's routing rules and route may have been removed by a prior `ifdown`; the `connected` arm recreates them unconditionally so the interface is fully routable before the policy re-selects it. On an ordinary link-negotiation event the routing is already in place and the recreate calls are harmless no-ops. The snap-back (`mwan3_flush_worse_metric_sticky`) returns new flows from lower-ranked same-family backups to the recovered interface; established flows on those backups are left to complete undisturbed. #### Automatic Gateway Tracking (`track_gateway`) @@ -1318,21 +1483,32 @@ config interface 'wan' The corresponding LuCI control is the "Track gateway" checkbox in the Interface tab (see [Section 15.1.2](#1512-interface)). -### 10.4 Interface Down (hotplug) +### 10.4 Interface Down and Soft Failure (hotplug) ``` netifd signals ifdown for $INTERFACE +--> 25-mwan3 hotplug script +--> mwan3_set_iface_hotplug_state "offline" - +--> mwan3_delete_iface_map_entries() flush sticky sets for this iface - +--> mwan3_delete_iface_rules() ip rule del - +--> mwan3_delete_iface_route() ip route flush table - +--> mwan3_delete_iface_nft() remove chain + jump rule + +--> [nft batch:] mwan3_set_policies_nft() + | mwan3_delete_iface_map_entries() + +--> mwan3_delete_iface_rules() ip rule del + +--> mwan3_delete_iface_route() ip route flush table + +--> mwan3_delete_iface_nft() remove chain + jump rule +--> procd_send_signal track_$INTERFACE USR1 - +--> mwan3_set_policies_nft() rebuild policies (failover) - +--> mwan3_flush_conntrack() flush conntrack entries for this iface's fwmark + +--> mwan3_flush_conntrack() flush this iface's ct entries ``` +``` +disconnected fires for $INTERFACE (tracker-detected soft failure, link up) + +--> 25-mwan3 hotplug script + +--> mwan3_set_iface_hotplug_state "offline" + +--> [nft batch:] mwan3_set_policies_nft() + | mwan3_delete_iface_map_entries() + +--> mwan3_flush_conntrack() flush this iface's ct entries +``` + +On `ifdown`, the policy is rebuilt before the interface's routing rules are removed, ensuring the policy drops the interface from the choosable set before its routes are gone; a packet is never tagged for an interface whose routing has already been torn down. On `disconnected`, the interface remains physically up so routing is left intact; only the policy and stickiness records are updated. In both cases the policy rebuild and sticky flush commit as one nft transaction, in that fixed order: the rebuild runs first so the newly built policy cannot re-seed the sticky records that the flush is about to clear. The per-interface conntrack flush follows outside the transaction as a separate operation on kernel state that nftables does not own, and immediately moves established flows off the failed interface rather than leaving them to hang until TCP retransmit timeouts expire. + ### 10.5 Stop ``` @@ -1341,16 +1517,14 @@ netifd signals ifdown for $INTERFACE +--> mwan3_interface_shutdown per interface trigger ifdown hotplug +--> flush ip routing tables (1..MWAN3_INTERFACE_MAX) +--> delete ip rules in configured ranges - +--> flush ALL mwan3_* chains (rules removed, skeleton chains kept) - +--> delete dynamic chains (iface_in_*, policy_*, etc.) - +--> final safety flush of skeleton chains - +--> flush ALL mwan3_* sets; flush and delete sticky maps +--> $NFT delete table inet mwan3 (complete removal) +--> rm -rf status dirs Result: table inet mwan3 no longer exists. Clean slate. ``` +The single table delete is the whole nftables teardown: one kernel transaction drops every chain, set and map mwan3 created. mwan3 previously dismantled those objects one at a time, flushing all its chains, deleting the dynamic ones, flushing its sets and deleting its sticky maps before removing what remained. That sequence dates from mwan3 operating inside `table inet fw4`, a table it did not own and could not drop; with its own table it is redundant work, and removing it is what makes stop fast. + ### 10.6 Conntrack Management mwan3 uses the Linux connection tracking table (conntrack) to persist routing decisions across the packets of a single connection. On the first packet of a new flow, mwan3 stamps the packet with an interface fwmark and saves that mark into the conntrack entry's ct mark field. All subsequent packets in the same connection restore the ct mark back to the packet mark at the start of the prerouting/output chain, bypassing policy re-evaluation entirely. This means that routing policy changes do not take effect for established connections until their conntrack entries are removed. @@ -1387,13 +1561,13 @@ When flow offloading is active, the kernel fast-paths established flows directly This does not apply to hardware flow offloading, which uses different kernel mechanisms and is not affected by writing to `/proc/net/nf_conntrack`. -**3. Interface down - selective flush (`mwan3_flush_conntrack`)** +**3. Interface down / soft failure - selective flush (`mwan3_flush_conntrack`)** -Immediately after `mwan3_set_policies_nft` rebuilds the policies for a failing interface, the hotplug script calls `mwan3_flush_conntrack`. This function has two independent paths that both run on `ifdown`: +After the policy is rebuilt for a failing interface, the hotplug script calls `mwan3_flush_conntrack`. This function has two independent paths that both run on `ifdown` and `disconnected`: First, the UCI `flush_conntrack` path: if the per-interface `flush_conntrack` UCI option contains `ifdown`, mwan3 writes `f` to `/proc/net/nf_conntrack`, flushing the entire global conntrack table. This is the legacy behaviour, retained for users who need it. Because it disrupts all connections on all interfaces simultaneously, it is not set by default. -Second, the selective flush path, which always runs on `ifdown` regardless of the UCI option: +Second, the selective flush path, which always runs on both `ifdown` and `disconnected` regardless of the UCI option: ``` mwan3ct flush --mark /$MMX_MASK @@ -1422,6 +1596,7 @@ This removes all mwan3-marked conntrack entries that have never received a reply | `start` | - | - | if flow_offloading | yes (always) | | `reload` | - | - | - | - | | `ifdown` | yes (always) | - | if `flush_conntrack` includes `ifdown` | - | +| `disconnected` | yes (always) | - | if `flush_conntrack` includes `disconnected` | - | | `ifup` | - | if `status=online` | if `flush_conntrack` includes `ifup` | - | | `connected` | - | yes (always) | - | - | | `stop` | - | - | - | - | @@ -1444,7 +1619,7 @@ The default conntrack management fits most deployments. Three cases where additi - **Load-balanced configurations:** when a failed WAN recovers, the automatic ifdown flush has already cleared entries for the failed interface, but all surviving flows remain pinned to the WANs that carried the load during the outage. They will not redistribute until they expire naturally. -- **Failover with a preferred primary:** when the primary WAN recovers, existing connections stay on the failover WAN until natural expiry rather than snapping back. In cost-sensitive setups where the primary is cheaper or unmetered, lingering on the failover WAN has a direct cost. +- **Failover with a preferred primary:** when the primary WAN recovers, existing connections stay on the failover WAN until natural expiry rather than snapping back. New connections return to the recovered primary on their own, since the snap-back clears the worse-metric sticky pins so the next new flow re-picks the higher-priority WAN, and where stickiness is not in use new flows re-pick by policy regardless; only established connections linger, which is the case this bullet addresses. In cost-sensitive setups where the primary is cheaper or unmetered, lingering on the failover WAN has a direct cost. - **Application of new policies / rules to *all* flows immediately:** a config save will cause all *new* connections be be immediately routed according to the new policy / ruleset, but will leave existing connections intact rather than interrupting them. Users can choose to override that behaviour and make the new policy take effect immediately even for existing connections, but at the cost of interrupting connections such as SSH, file downloads, etc. @@ -1557,7 +1732,11 @@ set youtube_v4 { } ``` -Placing `counter` inside the `flags` list causes an nft parse error. +Placing `counter` inside the `flags` list causes an nft parse error. The `auto-merge` line appears here only because this set has no `timeout`; a set with a `timeout` omits it (see auto-merge and timeouts below). + +### auto-merge and timeouts + +User-defined sets are declared with `flags interval`, and `auto-merge` is appended only when no `timeout` is configured. `auto-merge` coalesces any run of contiguous addresses into a single merged range, a useful memory optimization for static CIDR sets, but the merged range carries only one `timeout` for the whole run. On a timeout-bearing set populated host by host by an external resolver, for example a `list domain` set filled by dnsmasq `nftset=` (or smartdns), that coalescing discards the per-element TTLs: under the incremental insertion a resolver uses, the merged range keeps the timeout of whichever element most recently extended it, so a short-lived member extending a run of long-lived ones evicts the whole range early, and a long-lived member extending a run keeps a short-lived member alive well past its TTL. For resolver-populated sets the per-element TTL is the entire point, so `auto-merge` is dropped whenever a `timeout` is set. The `interval` flag is retained either way, so timeout sets can still hold CIDR entries; only the coalescing is dropped. ### dnsmasq Integration @@ -1808,6 +1987,8 @@ The Configuration tab (`configuration.js`) performs a static analysis of the mwa The Notify tab (`notify.js`) provides a raw textarea editor for `/etc/mwan3.user`. This file is sourced by the mwan3 hotplug handler at the end of interface up/down events, allowing site-specific shell commands to run in response to WAN state changes. The tab reads the file via `fs.read` on load and writes it back via `fs.write` on save; no UCI is involved. +The menu node `admin/network/mwan3/notify` declares an ACL dependency on `luci-app-mwan3-notify`, so the tab is absent entirely for a session that does not hold that group. Where the group is held read-only, `notify.js` sets `isReadonlyView` from `L.hasViewPermission()`, which suppresses `handleSave` and disables the textarea, leaving the hook visible but not editable. + ### 15.2 Status App The Status app provides live monitoring and diagnostic tools. All tabs that show runtime data poll automatically via the LuCI poll mechanism. @@ -1832,20 +2013,21 @@ For the selected interface, the tab shows a header card with status, tracking me The Routing Health tab (`routing.js`) polls `mwan3.routing_health` and performs a live sanity check of the ip rules and routing tables mwan3 maintains for each configured interface. -**Per-interface cards:** Each card shows the interface name, sequential index, and mwan3track-reported status, followed by four status rows: +**Per-interface cards:** Each card shows the interface name, sequential index, and mwan3track-reported status, followed by four status rows, five on IPv4 interfaces: - **IP rule (iif)** at priority `iif_base + index`: expected present when the interface is online. - **IP rule (fwmark)** at priority `fwmark_base + index`: expected present when the interface is online. - **IP rule (unreachable)** at priority `unreachable_base + index`: expected present when the interface is online. +- **IP rule (source)** at priority `src_base + index`: expected present when the interface is online. Rendered only for IPv4-family interfaces, the only ones for which `routing_health` reports a `src_rule` object. A device holding several global IPv4 addresses carries one rule per address at this priority. - **Routing table** (table ID = index): whether the per-interface routing table has a default route. -Each row renders a status badge: "Present" (green) or "Missing" (red) when the item is expected to be present (interface online); "Present (unexpected)" (yellow) or "Absent" (grey) when expected to be absent (interface offline); neutral grey "Present"/"Absent" for unknown or disabled interfaces. The card border is green when all four items are in the expected state, yellow for partial state, red when the interface is online and items are missing, and grey for offline or disabled interfaces in the expected state. +Each row renders a status badge: "Present" (green) or "Missing" (red) when the item is expected to be present (interface online); neutral grey "Present"/"Absent" for every other state, where presence is reported without judgement. The card border is green when all rows are in the expected state, yellow for partial state or for an interface reported offline or disconnecting, red when the interface is online and no rules are present, and grey for disabled or unknown interfaces. **Summary card:** A single line at the top reports total healthy, degraded, and failing interface counts, plus stale rule count if any. **Stale rules:** ip rules whose priority falls within the mwan3 priority ranges but do not correspond to any currently configured UCI interface are listed in a separate table. Stale rules indicate that mwan3 was not cleanly shut down after an interface was removed from the configuration. -**Field guide:** A reference panel at the bottom explains each field (Index, iif rule, fwmark rule, unreachable rule) in plain language, using the actual configured base priority values read from the `routing_health` response rather than hardcoded defaults. +**Field guide:** A reference panel at the bottom explains each field (Index, iif rule, fwmark rule, unreachable rule, source rule) in plain language, using the actual configured base priority values read from the `routing_health` response rather than hardcoded defaults. The source base is derived by the backend as `unreachable_base + iface_max + 1` rather than read from UCI; the fallback in `mwan3/constants.js` is 3061, the value the default bases and the default `0x3F00` mask imply. #### 15.2.4 IP Sets (Status) @@ -1897,11 +2079,13 @@ The `/usr/libexec/luci-mwan3` shell script provides backend commands invoked by #### ACL Permissions (`luci-app-mwan3.json`) -The rpcd ACL file grants the LuCI frontend permission to call rpcd methods and execute files. It defines two ACL groups: +The rpcd ACL file grants the LuCI frontend permission to call rpcd methods and execute files. It defines three ACL groups: -**`luci-app-mwan3`** (configuration access): Read permission for `mwan3.status`, `mwan3.nftset_info`, `mwan3.nftset_members`, `mwan3.nftset_elements`, `mwan3.resolve_host`, and the mwan3 and network UCI packages. Write permission for `mwan3.nftset_flush`, `mwan3.nftset_reload`, `mwan3.nftset_resolve`, and the mwan3 UCI package. Also grants `fs.stat` read access for detecting optional binaries (httping, nping, arping) and exec permission for `luci-mwan3 nftset dump`. +**`luci-app-mwan3`** (configuration access): Read permission for `mwan3.status`, `mwan3.nftset_info`, `mwan3.nftset_members`, `mwan3.nftset_elements`, `mwan3.resolve_host`, and the mwan3 and network UCI packages. Write permission for `mwan3.nftset_flush`, `mwan3.nftset_reload`, `mwan3.nftset_resolve`, and the mwan3 UCI package. Optional-binary detection needs two grants working together: the `list` permission on `/usr/bin/httping`, `/usr/bin/nping` and `/usr/bin/arping`, which is what rpcd's `stat` handler checks on the path, and the `stat` method on the ubus `file` object, which the HTTP ubus endpoint checks before dispatching the call. The read block also grants `cgi-io` `exec` and exec permission for `luci-mwan3 nftset dump`. -**`luci-app-mwan3-status`** (status and diagnostic access): Read permission for `mwan3.status`, `mwan3.nftset_info`, `mwan3.nftset_members`, `mwan3.nftset_elements`, `mwan3.resolve_host`, and `mwan3.routing_health`. Write permission for `mwan3.nftset_flush`, `mwan3.nftset_reload`, `mwan3.nftset_resolve`, and exec permission for the `luci-mwan3 diag` subcommands, `mwan3 internal`, `mwan3 ifup`, and `mwan3 ifdown`. +**`luci-app-mwan3-status`** (status and diagnostic access): Read permission for `mwan3.status`, `mwan3.nftset_info`, `mwan3.nftset_members`, `mwan3.nftset_elements`, `mwan3.resolve_host`, and `mwan3.routing_health`, together with `cgi-io` `exec` and exec permission for `/usr/sbin/mwan3 status`. Write permission for `mwan3.nftset_flush`, `mwan3.nftset_reload`, `mwan3.nftset_resolve`, the `exec` method on the ubus `file` object, and exec permission for the `luci-mwan3 diag` subcommands, `mwan3 internal ipv4` and `mwan3 internal ipv6`, `mwan3 ifup`, and `mwan3 ifdown`. + +**`luci-app-mwan3-notify`** (hotplug hook editing): Read permission for `/etc/mwan3.user` together with the `read` method on the ubus `file` object. Write permission for the same path together with that object's `write` and `remove` methods. The group is separate so that delegating mwan3 configuration does not imply the ability to edit the hook: `/etc/mwan3.user` is a shell script the hotplug handler runs as root on every interface event, so write access to it is equivalent to root. Before the split these grants sat in the configuration group. A session without the group does not see the Notify tab, read gives a read-only view, and write gives editing. --- @@ -1951,7 +2135,7 @@ Looks up the named set in UCI, collects `list domain` entries, sends SIGHUP to d Compares the UCI configuration against live kernel state. For each mwan3 interface (by 1-based UCI order index N): -- Checks for ip rule at priority IIF_BASE+N (iif), FWMARK_BASE+N (fwmark), and UNREACHABLE_BASE+N (unreachable) via rtnl RTM_GETRULE. Base values read from UCI globals, defaulting to 1000/2000/3000. +- Checks for ip rule at priority IIF_BASE+N (iif), FWMARK_BASE+N (fwmark), UNREACHABLE_BASE+N (unreachable), and SRC_BASE+N (source, IPv4 interfaces only) via rtnl RTM_GETRULE. Base values read from UCI globals, defaulting to 1000/2000/3000, with SRC_BASE derived as UNREACHABLE_BASE + iface_max + 1. - Checks routing table N for a default route via rtnl RTM_GETROUTE - Reads `/var/run/mwan3track//STATUS` for current online/offline state - Reports stale ip rules (priorities in mwan3's range with no matching UCI interface) @@ -2048,7 +2232,365 @@ Before printing any output the script builds a map of every public routable IPv4 ## 18. Changelog -### 18.1 Version 3.6.9 +### 18.1 Version 3.6.12 + +**Summary:** Adds IPv4 source-address return routing. Router-originated traffic sourced from a WAN's own address must leave by that WAN, but a sender that routes itself outside netfilter's view followed the main table out the default WAN and was dropped upstream or masqueraded to the wrong address. WireGuard is the case that motivates it, where replies to a handshake arriving on a non-default WAN left by the default WAN and the handshake never completed. One ip rule per global-scope address on each WAN now sends those lookups back to the arrival WAN. The rpcd status method reports the new rules and the web interface shows them alongside the existing three on its routing health view. + +A route wide enough to carry the whole internet without being a literal default is now treated as part of a default route when the bypass sets are built. VPN clients routinely install exactly that, OpenVPN with redirect-gateway def1 and split-tunnel WireGuard installing a pair of half-width routes rather than a default, and some IPv6 providers pushing the global-unicast prefix. One such route anywhere mwan3 scanned handed the entire address family to the bypass and silently stopped rule steering, sticky pinning, blackholing and inbound reply-path classification from working. Routes in the extra lookup tables are also no longer recorded in the connected-networks set, where a stale copy could outlive the route itself and keep a destination exempt from policy routing for hours. + +Three faults in rule building are corrected. An nft set referenced by rules of both address families but defined nowhere in the config was created twice with conflicting types, which nft rejected, leaving the router with no user rules at all. A sticky rule matching both families built its per-rule chain with IPv4 lookups only, so IPv6 flows were never pinned and stickiness silently applied to one family, and for ICMP the late IPv6 lookup produced a mark matching no interface at all. IPv6 policy members are no longer added on kernels without IPv6, where a policy with no usable IPv4 members steered IPv4 traffic to an IPv6 member instead of falling through to its last resort action. + +Three further faults affect the service itself. A service that started with every interface disabled never created its nft table, so a later config change reloaded against nothing and left the router with no ruleset until a manual restart, and now falls back to a full restart instead. The two route monitor instances shared a single nftables batch file and could race on it around WAN transitions and prefix renewals, leaving the bypass sets stale, and each now has its own. The route seeding helper counted an interface that omits the enabled option as enabled, where every other consumer of an interface section treats it as disabled, which withheld that device's routes from every other interface's table. + +The diagnostic report no longer prints public addresses that appear only in the nft ruleset or the log, service stop time is reduced by dropping tear-down code left over from the fw4 era, and the shipped base files now link to the nftables documentation rather than the legacy iptables page. In the web interface, address lists in table columns are reduced to their first entry and a count of the remainder, so a long comma-separated list no longer stretches or overflows the cell, and the address checks now call the framework's own parsers. + +--- + +### mwan3: update documentation links in shipped base files + +The default UCI config and the hotplug script both linked to the legacy iptables wiki page. Point them at the nftables page, whose sections describe the configuration options and hotplug environment that this package actually implements. + +--- + +### mwan3: keep extra lookup table routes out of the connected set + +Destinations with a route in one of the `rt_table_lookup` tables are exempt from policy routing. The route monitor recorded these routes in two places: the set that tracks them and the set that tracks directly connected networks. The first is updated whenever a route in those tables is added or deleted, but the second is only rebuilt when the main routing table changes. Deleting a route from an extra table therefore ended its exemption in one set while the copy in the other kept it alive, and traffic to that destination continued to bypass policy routing, for hours or days on a quiet router. + +Record only main table routes in the connected networks set, as the startup code already does. Nothing is lost by this: the set dedicated to the extra table routes grants the same exemption and is kept current. + +--- + +### mwan3: keep default-equivalent routes out of the bypass sets + +Destinations that `mwan3` should not steer are collected into bypass sets built by scanning the routing tables, and anything in those sets is marked for the main table before and after the user rules run. The scan recognises a default route only in its literal form, so a route wide enough to carry the whole internet without actually being a default looks like an ordinary network to it. VPN clients routinely install exactly that: OpenVPN with `redirect-gateway def1` and split-tunnel WireGuard install a pair of half-width routes and not a default, and some IPv6 providers push the global-unicast prefix. One such route anywhere `mwan3` scans hands the entire address family to the bypass, and rule steering, sticky pinning, a blackhole and inbound reply-path classification stop working silently. + +Consider a route at or below /2 for IPv4 and /3 for IPv6, or with no destination at all, to be a component of a default route. Nothing a real network hands out is that wide. Anyone who genuinely wants a broad bypass still has the `bypass_network` option. Routes are judged one at a time, so a client that has installed only half its pair is covered too. + +Only the bypass sets use the wider definition. Filling an interface's own table with routes, and taking that interface offline when its table holds no default both still require a literal default route. + +--- + +### mwan3: fix interface enabled in route seeding helper + +The interface route seeding helper read an interface section's `enabled` option with a default of 1, where every other consumer of an interface section defaults it to 0. The init script, the hotplug handler, the interface state paths and the route monitor all treat a section that omits the option as disabled. + +The defect is longstanding: the shell function this helper replaced called `config_get_bool` without its default argument, so a section with no `enabled` option left the variable empty and the numeric test meant to skip the interface failed on an empty string rather than skipping it. + +Counting the interface as enabled put its device into the device-to-table map, which withheld that device's routes from every other interface's table while delivering them to a table that is never built, because `mwan3` never brings up a disabled interface. Nothing relied on that. + +The visible consequence was that traffic in another interface's table addressed to a subnet directly connected to that device found no specific route and fell through to that table's default route. + +--- + +### mwan3: fix public addresses leaking into the mwan3-diag report + +Some public addresses were printed unmasked. The anonymiser replaces only addresses learned from neighbours, routes, rules, interface addresses, status and config, so an address appearing only in the `nft` ruleset or the log, such as a NAT map element, was printed as-is. + +Now the `nft` and log text the report prints is also swept for addresses before placeholders are assigned, so every public address in the report is masked. + +--- + +### mwan3: give each mwan3rtmon instance its own nft batch file + +`mwan3rtmon` runs as two long-lived instances, one per address family, and both wrote their nftables batches to the same fixed-name file, opened with truncation and never removed. When both families' routes changed at the same time, as is common around WAN transitions and prefix renewals, the two instances could race on the shared file: one could truncate or rewrite it while the other was writing or loading it with `nft -f`, so a load could apply a truncated, mixed or wrong-family batch, or fail outright, leaving the connected or custom destination sets stale until the next route change rebuilt them. The last batch file was also left behind after every load. + +Name the batch file with the instance's address family, which is sufficient because one instance runs per family and each instance is single-threaded, and remove the file on every exit path after it is created, matching the per-process, self-cleaning convention of the shell batch machinery in `common.sh`. + +--- + +### mwan3: create a missing set only once per rebuild + +A rule can match on an `nft` set that `mwan3` does not define in its own config, leaving it to be created in `table inet mwan3` by something else, or never created at all through a config error. `mwan3` creates an empty set when it finds one missing, as an unresolved set reference would break the atomic `nft -f` batch load. + +The address type came from whichever rule needed the set, so two rules matching the same missing set from different families each created it, with conflicting types. `nft` rejected the batch, leaving the router with no user rules at all. + +`mwan3` now remembers the sets it creates as it builds. The first rule to need a missing set settles its address type, while a later rule of the other family is skipped, quietly if it is a `family=any` rule, or with a warning if the rule is family specific. + +--- + +### mwan3: skip IPv6 policy members on kernels without IPv6 + +Ensure that IPv6 policy members are only added when the kernel supports IPv6. Such members previously bypassed member selection but were still added to the chain, so a policy with no usable IPv4 members steered IPv4 traffic to an IPv6 member instead of falling through to its `last_resort` action. + +--- + +### mwan3: complete sticky chains for family=any rules + +A sticky rule with `family=any` and no family-specific match element is rendered once, as a single line matching both address families, but the per-rule sticky chain it jumps to was built with IPv4 sets, lookups and updates only. IPv6 flows matching the rule entered the chain, missed every lookup and were never pinned: the policy re-evaluated each new flow, so connections from one source could egress different WANs. Stickiness silently applied to IPv4 only. + +For `proto=icmp` such a rule renders on both family passes and each pass appended its own family's sticky arm to the shared chain. The IPv6 lookups landed after the IPv4 arm's fall-through, so pinned IPv6 sources were re-marked by the policy first and the late lookup then ORed the old pin into the new mark, producing a value matching no interface. + +Fix both by building the per-rule chain exactly once, complete for every family whose traffic can reach it: all lookups, then the single fall-through to the policy, then all update rules. Rules with an explicit family or a family-specific match element render as before. + +--- + +### mwan3: report IPv4 source rules in routing_health + +The `routing_health` method reported the iif, fwmark and unreachable `ip rules` per interface but not the IPv4 source-address return routing rules, so a consumer could neither display them nor factor them into an interface's health. + +For each IPv4-family interface, add a `src_rule` object carrying the rule's presence and priority (unreachable base + interface maximum + 1 + index), and extend `rule_bases` with the source base. Presence is tested against a family-filtered index of the IPv4 rule dump so an unrelated IPv6 rule sharing the priority is not counted. A device with several global IPv4 addresses holds one rule per address at this single priority, so presence means the slot is populated rather than that every address is covered. IPv6-family interfaces carry no `src_rule` key. + +--- + +### mwan3: fix whitespace in rpcd module + +Adjust blank-line spacing around several comment blocks in the `rpcd` module for consistency with comment formatting elsewhere, and remove one blank line between two variable declarations. + +--- + +### mwan3: restart on reload when the nft table is missing + +When the service starts with every interface disabled, `start_service` returns before creating the `nft` table, yet the status directory makes it report running, so a config change drives a reload that rebuilds against the missing table. The batch fails and `mwan3` is left with no ruleset until a manual restart. + +Guard `reload_service` with a table-existence check and fall back to a full restart when the table is absent. + +--- + +### mwan3: optimise service stop time + +Optimise `mwan3` service stop time by removing `table inet fw4`-era manual tear-down code. + +--- + +### mwan3: add IPv4 source-address return routing + +Router-originated IPv4 traffic sourced from a WAN's own address must leave by that WAN, but a sender that routes itself outside netfilter's view follows the main table out the default WAN and is dropped upstream or masqueraded to the wrong address. + +WireGuard is a good example: replies to a handshake arriving on a non-default WAN leave by the default WAN and the handshake never completes. Locally originated flows that no user rule classifies share the gap. + +Install one `ip rule` per global-scope IPv4 address on each WAN, `from lookup `, on the same lifecycle as the interface's existing rules. Route lookups then resolve back to the arrival WAN and the reply keeps the correct source. Marked traffic still outranks these rules, so policy verdicts and the fail-closed backstops behave as before, and the rules are inert for forwarded traffic and single-WAN setups. + +Update the `rpcd` module to recognise the new source rules, which its routing health check would otherwise report as stale. + +--- + +### luci-app-mwan3: fix ACL grant for optional binary detection + +Detection of the optional tracking binaries runs through `fs.stat`, whose ubus method was not granted by the configuration group. Sessions holding only this app's ACL groups therefore offered ping as the only tracking method even with the binaries installed. Grant the `stat` method. + +--- + +### luci-app-mwan3: replace custom validator calls with framework calls + +The rule and IP set address checks ran the `ipaddr` datatype through a custom validator object. + +Add `ipAddrOrPrefix()` and `ipv6Cidr()` to the validators module, both built on the framework's `parseIPv4()` and `parseIPv6()`. The three address checks now call `ipAddrOrPrefix()`. The custom validator stays for the types the framework publishes no parser for. + +--- + +### luci-app-mwan3: summarise address lists in table columns + +The source and destination address options accept a comma-separated list, and every table that previews a rule rendered that list in full. A comma-joined list is a single long token with nothing to break on, so it stretches an auto-layout table column and overflows the cell of a fixed-layout one. + +Add a shared formatter that reduces such a list to its first entry followed by a count of the remainder. The stored value is untouched and the editing widgets still show every entry, so the complete list remains available in the rule modal. + +The source preview is reshaped to build its text from a list of parts, matching the destination preview directly below it, which already does so because it appends a mark element. + +The translation template is regenerated. The only changes are moved source line references, not strings. + +--- + +### luci-app-mwan3: regenerate translation template + +The translation template had not been regenerated since the last change that added strings to it. Regenerate it from current sources. No functional change. + +--- + +### luci-app-mwan3: show IPv4 source rules on Routing Health + +The `routing_health` backend now reports the IPv4 source-address return routing rules, but the Routing Health tab rendered only the iif, fwmark and unreachable rows and ignored the source band. + +Render a fourth "IP rule (source)" row on IPv4-family interface cards, keyed off the `src_rule` object so it never appears where the concept does not apply, and fold the source rule into the online health judgement so a missing rule degrades the card. Document the rule in the field reference, covering its purpose, lifecycle, the one-rule-per-address behaviour and its IPv4-only scope, and update the closing summary for the fourth row. Add a `src` fallback to the shared rule-base defaults for when the backend does not report bases. + +--- + +### 18.2 Version 3.6.11 + +**Summary:** Fix the nft rule rendering order such that it preserves strict config order. + +--- + +### mwan3: render user rules in config order + +The user rule renderer looped over address family first and the config rules second, so the installed `mwan3_rules` chain held the entire IPv4 render pass followed by the entire IPv6 render pass. Config order was preserved within each pass but not across the pass boundary. + +A `family=any` rule with no IP-version-specific match element (for example a bare protocol and port rule, or an fwmark rule) is emitted once, on the IPv4 pass, as a single family-agnostic line that also matches IPv6 traffic at runtime. Because the whole IPv4 pass rendered first, that line always landed ahead of every IPv6-only rule in the chain regardless of its position in the config, and shadowed any IPv6 rule the configuration placed before it. Rule evaluation is first match, so no config ordering could place an IPv6-only rule ahead of a family-agnostic one, and reordering rules in the config silently failed to change evaluation order. + +Invert the loop nesting: iterate the config rules in order and render each rule's IPv4 and IPv6 variants at that rule's position. The per-rule, per-family emission function is unchanged, so every rendered line is identical; only the order of the lines in `mwan3_rules` changes, and it now always matches config order. The chain is a flat first-match list with no structural dependency on the old pass grouping, and per-rule sticky chains and sets are keyed by rule and family, so their content is unaffected. + +For configurations that interleave family-agnostic rules with family-specific ones, evaluation order now follows config order strictly. + +--- + +### 18.3 Version 3.6.10 + +**Summary:** This release corrects six longstanding bugs, all present in the original codebase, fixes some privilege escalation issues, corrects a tracking-health edge-case, modularises and refactors the LuCI app for maintainability and to make exclusive use of LuCI's published API, and makes ucode scripts log to syslog. Normal operation while every internet connection is healthy is unchanged throughout. + +Four of the changes address what happens at the moment a connection fails, recovers, or is brought up or down. When a connection was detected as failed while still physically connected, the records of which sessions and devices were using it were not cleared, so traffic stayed pointed at the dead connection for minutes instead of moving to a working one. + +When a preferred connection later recovered, traffic that had diverted to a backup did not return to it. Both are now corrected: on failure the records are cleared immediately, and on recovery new traffic returns to the better connection while sessions already in progress finish where they are. + +The order in which routing and connection selection are updated during a transition was also wrong in both directions: on recovery, a connection was made available for traffic before its route was rebuilt, so traffic was assigned to it before it could carry any. On shutdown, a connection's route was removed before traffic stopped being assigned to it. Both orders are now correct. + +A per-connection safety block that rejects traffic with no working route was being removed when a connection went down, at the exact moment it was needed. It is now kept in place for as long as the service runs, so traffic with no route is rejected rather than escaping over the default internet link. + +All forwarding-rule changes during a transition are now applied as a single all-or-nothing update, closing a brief window in which the rules were only partly in place and traffic could be steered by an incomplete set of them. + +A separate fix removes a fallback rule that silently ran in place of a policy's configured behaviour for when all its connections are down, sending traffic out the default link even when the policy was explicitly set to block it. + +The sixth fix corrects corruption of per-entry expiry times in address sets populated by a DNS resolver: a kernel range-merging optimisation was discarding the individual expiry of each address and replacing it with the expiry of whichever address had last extended the merged range, causing entries to expire too early or be kept alive too long. + +Three further fixes harden the service against a user who has been granted only limited, delegated control of the mwan3 configuration, closing routes by which that restricted role could take over the whole router or another user's web session. Two of them address ways such a user could gain administrator-level command execution: one web-interface permission let them edit a script that mwan3 runs as the administrator on every connection event, and two internal helpers built system commands out of configuration values, such as a domain name or an address-set entry, without first checking them. That editing permission is now separated into its own group that ordinary configuration access no longer carries, and every value is checked before it reaches a command. The third fix escapes configuration text before it appears on the status pages, so text saved by one user can no longer run as code in the browser of anyone who later views those pages. + +Several further, unrelated changes round out the 3.6.10 release. A missing default route on an interface's own routing table is now treated as a tracking failure, so an interface that its tracker would otherwise still report as online is taken offline instead of blackholing the traffic steered to it. The routing-health view in the web interface has been updated to match current behaviour, in which the per-interface reject rule is a permanent fixture rather than one that appears and disappears with the interface, so its presence on an offline interface is now shown as expected rather than as a fault. The web interface itself has also been reworked to build on LuCI's standard, supported components and split into shared modules for easier maintenance, with no change to how it behaves aside from a few small display corrections. Finally, the ucode components now log to the system log with a per-component tag, matching the shell components. + +--- + +### mwan3: validate ipset elements and resolver hosts before use + +The `rpcd` backend runs as root. Two of its methods built command lines from UCI-derived values with no validation and ran them through `popen`, executing via a shell fork. `nftset_resolve` passed each domain straight to the `nslookup` command line and `nftset_reload` joined the section's entry values and the lines of a loadfile into an `nft add element` command line. A user holding only a delegated config role has UCI `mwan3` write plus these methods, so a crafted domain, entry, or loadfile line could potentially run arbitrary commands as root. + +The same entry list is consumed a second time by the ordinary ipset apply path, which builds an `add element` batch that is later committed with `nft -f`. There a crafted value could close the element braces early and inject rogue nftables statements, executed as root on the next apply. + +Reject anything that is not a plain hostname before it reaches `nslookup`. Validate every ipset element, from both the entry list and the loadfile, ensuring it is a valid IPv4 or IPv6 address, a CIDR, or a hyphen range, and drop anything else before it reaches a command line. Apply the element check both in the ucode and the shell path. + +--- + +### mwan3: log ucode components to syslog + +Route the ucode logging through a new shared logging function housed in `common.uc` that writes to syslog with a per-component tag, just like the shell components do via the `logger` tool. + +Add a `ucode-mod-log` dependency. + +--- + +### mwan3: take an interface offline when its table has no default route + +An interface's tracker reports it online on the strength of its probes, which reach their target over a connected or point-to-point route and do not need a default route. So an interface whose own routing table holds no default route still reports online: `mwan3` marks traffic to it, its fwmark `ip rule` sends that traffic to the interface's table, the table has no default, and the per-interface unreachable backstop drops it. The interface black-holes every flow steered to it while still showing online. + +Make a missing default route count as a tracking failure. The route monitor, which already maintains each interface's own table, checks whether that table holds a default route via the interface's device and writes up or down to a small file in the tracker's status directory. The tracker reads it once per round and, when it reads down, skips the probe so the round fails; the existing score and hysteresis then carry the interface offline, and back online when the route returns, with no new state and no new hotplug arm. The check is on the interface's own table, not the main table, because the fwmark rule looks up exactly that table, and it is recomputed from a full route dump rather than a single route event, so an interface that never held a default also reads down. An absent file is treated as up, so a tracker with no route monitor is unaffected, and the unreachable backstop is unchanged, so traffic still marked to a downed interface fails closed rather than leaking. + +--- + +### mwan3: batch each failover transition into one nft transaction + +When a WAN's usability changes, whether it stops being usable (a tracker-detected soft failure or an administrative or carrier down) or becomes usable again (a tracker-detected recovery, or coming back up while already online), `mwan3` rebuilds its routing policy, and on a teardown also clears the per-source stickiness records that kept traffic on the failed WAN. Until now `mwan3` issued these as several separate firewall operations. + +Because they were separate, the firewall could be observed in an intermediate state during the transition: a packet could be classified against a policy that had been flushed but not yet fully rebuilt, and on a teardown the stickiness records were cleared in a step that could briefly disagree with the rebuilt policy. These windows are sub-second, but they are real and serve no purpose. + +Apply the policy rebuild as a single firewall transaction on every transition, both the teardown paths and the recovery paths, committed in one step. The kernel applies the whole transaction at once, so traffic is never steered by a half-built policy. The resulting ruleset is identical to what the separate operations produced; only the number of kernel transactions changes. + +On a teardown the stickiness clearing joins the same transaction, in the fixed order of the policy first and the stickiness second, because the stale stickiness points at a WAN that is going away and must be cleared in the same step that drops it from the policy. On a recovery the stickiness reset that returns new traffic to the recovered WAN stays a separate step after the rebuild: it only resets records pointing at lower-priority WANs that are still working, so a connection that consults one in the instant before the reset still reaches a working WAN, and its timing is not safety-critical. + +The connection-tracking cleanup stays a separate step at the end of the transition, because connection-tracking marks are kernel state outside the firewall ruleset and cannot share its transaction. + +This corrects longstanding behaviour rather than a regression; the separate-operation form predates `mwan3`'s use of atomic firewall transactions. + +--- + +### mwan3: install static per-WAN unreachable backstops + +`mwan3` sends a connection out a chosen WAN by tagging it with a firewall mark and adding an `ip rule` that routes that mark to the WAN's table. Next to each of those it also installs a backstop rule that rejects the mark as unreachable when the WAN has no usable route. That backstop is what keeps traffic tagged for a WAN from quietly slipping out the main routing table, and the default route, when the WAN cannot carry it. + +The bug was that the backstop was added and removed together with the WAN's routing rules, so it only existed while the WAN was up. It was therefore missing at the worst possible moment: right after a WAN goes down, or during the split second its rules are torn down and rebuilt, a connection still tagged for that WAN had nothing to reject it and leaked out the default route. On a policy meant to be a hard boundary, such as a VPN, that is a genuine leak. + +The fix makes the backstops permanent. `mwan3` now installs one for every configured WAN when the service starts and on every reload, and removes them only when the service stops. The rules that actually route into a WAN still come and go with it, because they need the live interface, but the reject rule stays put. Traffic tagged for a WAN with no matching route, whether the WAN is down, disabled, or briefly mid-rebuild, is now rejected instead of leaking. This covers every configured WAN, including disabled and offline ones, since a WAN disabled while its connections are still live leaves tagged traffic that has to be caught. + +The backstops use only real WAN marks, never `mwan3`'s reserved marks, so a policy's `last_resort` setting is untouched: `last_resort "default"` still deliberately uses the main routing table, `"unreachable"` still rejects, and `"blackhole"` still drops. The new rules are created through the same helper that already builds `mwan3`'s global rules, and the existing stop-time cleanup already removes them. This corrects long-standing behaviour carried over from the old iptables-based `mwan3`; it is not a regression. + +--- + +### mwan3: clear stale WAN pins on soft failover and snap back on recovery + +When a WAN goes up or down, `mwan3` has to bring two caches back into step with the routing policy it rebuilds. Both caches decide a packet's WAN before the policy is consulted, and both can override it: the sticky sets, which tie a source address to a WAN without checking that the WAN is still up, and the connection marks kept by `conntrack`, which tie an existing connection to the WAN it started on. Both refresh themselves as traffic flows, so once either points at a WAN the policy has dropped, the mistake lasts for minutes rather than for the moment of the change. + +On a soft failure, where the tracker judges a WAN dead but the interface itself stays up, `mwan3` dropped the WAN from the policy but cleared neither cache. A source pinned in the dead WAN's sticky set was still tagged for it, and a connection already marked for it was still restored to it, so traffic either black-holed on the dead WAN or, when that WAN's routing entry was already gone, escaped out the main routing table's default route. Every leaked packet re-armed the sticky timer, so the wrong state held itself open. A hard interface-down cleared the affected connection marks; the soft failure cleared nothing. + +Now both caches are cleared on the soft-failure event, after the policy is rebuilt, in the order a teardown needs: policy first, then the sticky sets, then the connection marks. A sticky entry can stamp its WAN onto a connection's mark, but a connection's mark never feeds back into the sticky sets, so clearing in this order leaves nothing already-cleared to be re-dirtied. The targeted connection-mark flush that a hard interface-down already performed is now applied to the soft failure too. A pinned source picks the survivor on its next new connection, and an existing connection takes one reset and reconnects on the survivor instead of hanging until the dead WAN returns. Protocols that can move a live connection by themselves, such as `QUIC`, carry on without a reset. + +On recovery, new traffic is moved back off the backups the returning WAN outranks. `mwan3` ranks a policy's WANs by metric, lower winning, and keeps IPv4 and IPv6 as two separate contests, so a recovering WAN competes only with others of its own address family. When a higher-ranked WAN comes back, the sources parked on a lower-ranked backup of the same family should return to it. For each policy the recovered WAN belongs to, `mwan3` clears the sticky sets of that policy's lower-ranked, same-family WANs, and only for the rules that use the policy, so a parked source re-picks on its next new connection while connections already running on a backup are left alone. The narrowing matters in two ways. Keeping to the WAN's own family means an IPv4 recovery never disturbs IPv6 stickiness, or the reverse. Keeping to the policy and its rules means a WAN that is a backup in a failover policy and an equal-ranked WAN in a separate load-balancing policy keeps its load-balancing stickiness, which clearing by WAN alone would wrongly wipe. In a load-balancing policy every WAN ranks equally, so nothing is cleared and steady-state stickiness is kept. + +These are longstanding behaviours carried over from the old iptables-based `mwan3`: the sticky scheme never checked whether a WAN was still up, and the soft-failure path never cleared either cache. + +--- + +### mwan3: restore a WAN's routing on recovery and rebuild policy before teardown + +When a WAN changes state, `mwan3` has to update both the routing policy and that WAN's own `ip rules` and route. The handler that does this updated them in an order that could leave a packet tagged for a WAN whose routing was not in the matching state. + +When the tracker brought a WAN back up, the handler rebuilt the policy but, unlike a normal interface bring-up, never recreated that WAN's own routing rules and route. A WAN restored this way was added back to the policy and had traffic tagged for it, yet nothing routed that traffic until the next reload, so it was misrouted or dropped. The recovery path now recreates the WAN's rules and route before rebuilding the policy, the same as a normal bring-up does, so a recovered WAN can carry traffic the instant it is tagged for. Re-creating the rules is harmless when they already exist, so a recovery that races a WAN that is already up just re-asserts the same rules. + +When a WAN was taken down, by an administrator or by losing carrier, the handler removed that WAN's routing and cleared its sticky sets before rebuilding the policy. In the gap between removing the routing and rebuilding the policy, the still-stale policy could tag traffic for the downed WAN, whose routing rule was already gone, and a packet crossing that gap re-filled the sticky set that had just been cleared. The down path is reordered so the policy is rebuilt first, dropping the WAN before its routing is removed, and the sticky sets are cleared after the rebuild, so the rebuild can no longer re-fill a set it just cleared. + +Both behaviours are longstanding, carried over from the old iptables-based `mwan3`. Steady-state routing and the built ruleset are unchanged; the difference shows only across a change of state. + +--- + +### mwan3: remove offline-WAN fallback that overrode last_resort + +When a policy had no WANs online, the policy builder added an extra rule just ahead of the `last_resort` rule, as part of a broader link-selection change. It matched traffic that would have left through an offline WAN's device and tagged it for the main routing table. Because it sat before the `last_resort` rule, and both act only while the firewall mark is still unset, it ran instead of `last_resort` for that traffic. + +This silently overrode a deliberate setting. An operator who sets `last_resort` to `unreachable` or `blackhole` is telling the policy to fail closed when none of its WANs is available. The fallback instead sent some of that traffic out the main table. It picked that traffic by which device it would leave through, not by where it was going, so during a soft failure, where the failed WAN's device is still present and its default route is still in the main table, it also caught ordinary default-routed traffic, not just traffic to directly reachable hosts, and sent it back out the dead WAN via the main table. For a one-WAN policy used as a kill switch, such as a router-originated VPN tunnel with `last_resort unreachable`, this defeated the fail-closed behaviour for exactly the traffic the operator meant to block. + +Removing the fallback does not change reachability of directly connected or local destinations. Those are sent to the main routing table by the connected-destination and custom-destination checks that run before the policy, so that traffic never reaches the policy and never relied on this rule. The connected set is built from the main table's non-default network routes, so it already covers directly connected subnets and other non-default routes. The fallback's only remaining effect was on default-routed traffic leaving through a fully-down policy's offline WAN, which is exactly the case that should obey `last_resort`. + +The change is narrow. With `last_resort default` the policy already tags all its traffic for the main routing table, so nothing changes. Only `last_resort unreachable` and `blackhole` are affected, and only for default-routed traffic leaving a fully-down policy's offline WAN, which now fails closed as set. A destination reachable only by a host route out the offline WAN is not in the connected set and now obeys `last_resort` too; deliberate exceptions are still available through the custom-destination set. + +The now-unused offline-device bookkeeping in the per-WAN policy walk is removed with it. + +Fixes: `46ed09408e55` ("mwan3: refine link selection") + +--- + +### mwan3: disable auto-merge on timeout-bearing config ipsets + +Config ipset sections are declared as `interval` sets and `auto-merge` has been applied unconditionally. When such a set also carries per-element timeouts and is populated host by host by an external resolver, for example `dnsmasq` `nftset=` or `smartdns`, `auto-merge` coalesces any run of contiguous host addresses into a single range. The merged range carries one `timeout` and the per-element timeouts of the other members are discarded. Under the incremental insertion a resolver uses, the surviving timeout is that of whichever element most recently extended the range: a short-lived member that extends a run of long-lived members evicts the whole range early when its TTL fires, and a long-lived member that extends a run keeps a short-lived member alive well past its TTL. For resolver-populated sets the per-element TTL is the entire point, so this silently corrupts membership. + +`auto-merge` remains a useful memory optimization for static CIDR sets, so restrict it to sets without a `timeout`. The `interval` flag is retained so `timeout` sets can still hold CIDR entries; only the `auto-merge` coalescing is dropped. + +--- + +### luci-app-mwan3: move mwan3.user editing to its own ACL group + +The configuration ACL group granted read and write on `/etc/mwan3.user`, along with the ubus file read, write and remove methods. That file is a shell script run as root on every `mwan3` hotplug event, so anyone delegated the configuration role could edit it and run arbitrary code as root. That role therefore implied full root. + +Move the `/etc/mwan3.user` grants and the ubus file read, write and remove methods into a new `luci-app-mwan3-notify` group, and make the Notify menu node depend on it. The configuration group keeps only the ubus file list method and the `httping`, `nping` and `arping` list entries it needs for the tracking-binary presence check. A configuration delegate now gets no access to the hook by default. Read on the new group lets them view it read-only, and write lets them edit it. The Notify page also hides its Save button when the session lacks write. + +--- + +### luci-app-mwan3: escape config-derived values in the status views + +LuCI's `E()` and `dom.append` assign a bare string child to a node's `innerHTML`, where it is parsed as HTML, but append an array child or a node as escaped text. The `mwan3` status and analysis views passed UCI-derived free-form values straight in as bare strings. A user who can write `mwan3` UCI could store markup in any of these that then runs in the browser of anyone who later opens the status pages. + +Add a small helper that returns a text node, change the shared inline primitives to append their text as an array child so it is never written to `innerHTML`, and pass each of these values through the helper. The troubleshooting view's diagnostic output is likewise appended as an array child, which both escapes it and keeps it verbatim. + +--- + +### luci-app-mwan3: refactor and modularise + +App-wide refactor to use the published, supported client-side API throughout and modularisation to support maintainability. + +Logic that had accumulated as copies across the individual views is factored into a small shared library that the views compose from: render helpers, value formatters, address and set arithmetic, input validators, the interactive widgets, and the default constants. Checks and defaults that were restated in several places now have a single definition. The result is much less code per view and a smaller footprint for future changes. + +Visible behaviour and the configuration model are unchanged, aside from a few incidental display corrections. + +--- + +### luci-app-mwan3: update routing health display + +`mwan3` now installs the per-interface unreachable rule as a static backstop: it is created for every configured interface when the service starts and removed only when the service stops. It is therefore present regardless of whether the interface is online, offline or disabled, and it survives a hard `ifdown`. The routing health tab historically treated that rule as one that comes and goes with the interface, so it presented the now-normal presence of the backstop on a downed interface as exceptional. + +Also correct how the tab describes a soft tracker failure: only a hard `ifdown` removes the iif and fwmark rules and flushes the table. + +Update the display to reflect the actual rule lifecycle: + +- Presence of rules on an offline interface is expected. +- Show a disconnecting event as yellow-bordered. +- Update the field guide: a soft tracker failure retains all of the interface's rules; a hard `ifdown` removes the iif and fwmark rules and flushes the routing table, leaving intact the unreachable rule. + +--- + +### 18.4 Version 3.6.9 **Summary:** Stop router-originated IPv6 link-local traffic, most visibly odhcpd's DHCPv6 replies to LAN clients, from being policy-routed into a WAN routing table and failing, by accepting single-link scopes before classification. Clear an inherited skb mark on ingress so a tunnelled-WAN reply is no longer pinned to the wrong routing table. Emit dnsmasq `nftset=` directives correctly when a domain is shared across multiple ipset sections, and stage the fragment atomically under the runtime directory. @@ -2096,7 +2638,7 @@ The existing ICMPv6 ND bypass stays: NS/NA can also target global unicast addres --- -### 18.2 Version 3.6.8 +### 18.5 Version 3.6.8 **Summary:** Fix an operator precedence regression in 3.6.7 in the address family guard. @@ -2108,7 +2650,7 @@ The address-family guard in `mwan3_set_user_nft_rule()`combined its tests as `[ --- -### 18.3 Version 3.6.7 +### 18.6 Version 3.6.7 **Summary:** @@ -2272,7 +2814,7 @@ Improve the configuration checker's rule shadowing detection to recognise when t --- -### 18.4 Version 3.6.6 +### 18.7 Version 3.6.6 **Summary:** A race condition in the ct mark save logic has been fixed. The previous two-step clear and set sequence first cleared the MMX bits to zero and then set the new value in a consecutive rule, but between those two rules, under the right conditions, a packet on another CPU core could read the intermediate zero state, causing connections to be re-evaluated by the rules chain instead of being pinned to their assigned WAN. The clear is now folded into each setter chain as a single atomic expression, eliminating the race window and ensuring clear-set atomicity. @@ -2306,7 +2848,7 @@ The standalone clear rules in the prerouting and output chains are removed since --- -### 18.5 Version 3.6.5 +### 18.8 Version 3.6.5 **Summary:** Replaces all `ip` command output parsing with direct `ucode-mod-rtnl` netlink calls via four new helper scripts in `/lib/mwan3`: `mwan3-create-iface-route.uc`, `mwan3-get-addr.uc`, `mwan3-list-routes.uc`, `mwan3-manage-rules.uc`. These scripts replace the shell invocations of the ip binary and consequent (fragile) parsing of the ip output using `sed`, `awk` and `grep`, helping to make mwan3 more robust and future proofing it against potential changes in the output format of ip commands, since the helper scripts return precisely the information needed by mwan3 and do not require any parsing of the output. The scripts also contribute to a substantially enhanced efficiency profile, since there is now only one call per operation instead of multiple `ip` subprocesses each invoking `ip`, `sed`, `grep` and `awk`. @@ -2433,7 +2975,7 @@ Add exclusions for `0.0.0.0/8` and `224.0.0.0/3` so that these non-routable addr --- -### 18.6 Version 3.6.4 +### 18.9 Version 3.6.4 **Summary:** Fixes a regression in 3.6.3 that prevents mwan3's hotplug handler from running when the service is started but not explicitly enabled @@ -2447,7 +2989,7 @@ The guard was intended to prevent hotplug side effects when mwan3 is disabled, b --- -### 18.7 Version 3.6.3 +### 18.10 Version 3.6.3 **Summary:** Correctness fixes for edge cases and incomplete feature implementations: fixes ipset and ICMP protocol translation issues when rules use family=any; rebuilds ip rules and routing tables on reload to handle interface reordering and family changes; completes track_gateway support that was missing from two tracking status checks; adds runtime configuration reload to mwan3track via a SIGHUP handler and fixes hotplug handler initialization issues and boot race conditions. @@ -2527,7 +3069,7 @@ Remove the running check that followed `mwan3_init`. It was dead code: `mwan3_in --- -### 18.8 Version 3.6.2 +### 18.11 Version 3.6.2 **Summary:** A set of defensive edge-case fixes and correctness improvements. Binds the mwan3rtmon route listener before the initial netlink dumps to close a narrow startup race window, and replaces `main_route_cache` with an on-demand kernel query to eliminate a class of cache-drift failures that could only manifest if route events arrived during the dump phase. Aligns shell and mwan3rtmon custom-set filtering so both paths apply identical exclusions. Adds a dormant `is_default_route` guard to `populate_connected_set` as a forward-compatibility precaution. Clamps `check_quality` to 0 when the configured track method cannot produce quality samples, preventing an arithmetic error in the unusual case where `check_quality` is paired with a non-ping method. Fixes `mwan3_track_clean` which targeted incorrect paths and was a no-op, and tightens ip rule deletion at `stop_service` to use content-based matching rather than a priority-range regex, which matters only when rule bases are configured outside the default 1000-3999 band. @@ -2616,7 +3158,7 @@ To make the gates trustworthy at stop time, `mwan3_init` now persists `iif_rule_ --- -### 18.9 Version 3.6.1 +### 18.12 Version 3.6.1 **Summary:** Extends the legacy mwan3 custom sets that were previously only loaded statically during `start_service()` and `reload_service()` from the tables defined in the UCI global config list option `rt_table_lookup` to be fully dynamic, using mwan3rtmon to listen for and to add and remove routes from the custom sets in response to `RTM_NEWROUTE` and `RTM_DELROUTE` events on the tables defined with `list rt_table_lookup `. Adds a `SIGHUP` handler to mwan3rtmon to cause it to flush and repopulate these custom sets, ensuring that their contents remain in sync with any newly added or removed `list rt_table_lookup ` options in the mwan3 config. @@ -2642,7 +3184,7 @@ Add `handle_custom_set_event()`, called from `handle_route_event()` whenever a r --- -### 18.10 Version 3.6 +### 18.13 Version 3.6 **Summary:** Version 3.6 adds three user-visible features to mwan3 rules. Rules now support an `fwmark`/`fwmask` option to match packets by meta mark using a masked comparison, working alongside or instead of address and ipset matching; mwan3 logs a warning if the fwmask overlaps its internal `MMX_MASK` since such a mask would match packets already carrying an mwan3 classification mark. The ip rule priority tiers for per-interface rules are now configurable via three new globals UCI options (`iif_rule_base`, `fwmark_rule_base`, `unreachable_rule_base`), shifting from the fixed 1000/2000/3000 defaults; two ordering constraints are enforced at startup and rule deletion is rewritten to use content-based matching so it remains correct across base or `mmx_mask` changes. Rules gain `option enabled 0/1`, consistent with interfaces, ipsets, and members. @@ -2803,7 +3345,7 @@ The Policy assigned column label is shortened to Policy. --- -### 18.11 Version 3.5.3 +### 18.14 Version 3.5.3 **Summary:** Version 3.5.3 adds two major LuCI features and a set of bug fixes and routing reliability improvements. @@ -2893,7 +3435,7 @@ The address family selector controls A vs AAAA record resolution; IPv4 is prefer --- -### 18.12 Version 3.5.2 +### 18.15 Version 3.5.2 **Summary:** Version 3.5.2 is a bug-fix and maintenance release. It corrects a misrouting bug where kernel-generated NDP Neighbor Solicitation probes entered `mwan3_output` without a conntrack entry, fell through to `mwan3_rules`, and received a WAN policy mark that caused the kernel to probe the gateway via the wrong interface, cycling the NDP entry to FAILED state and breaking WRAP ping tracking for that interface. It updates the package dependency from `ip` to `ip-full` to ensure the full iproute2 implementation is always present, since the busybox `ip` is a minimal subset that does not support all options mwan3 requires. It adds `mwan3-diag`, a ucode diagnostic script installed to `/usr/sbin/mwan3-diag` that collects a comprehensive snapshot of mwan3 state -- interface status, policy routing rules, nftables ruleset, routing tables, conntrack summary and system log -- with all public IP addresses anonymised with stable placeholders so output can be shared safely. @@ -2919,7 +3461,7 @@ Add an icmpv6 NDP accept rule at the top of mwan3_output, mirroring the equivale --- -### 18.13 Version 3.5.1 +### 18.16 Version 3.5.1 **Summary:** Version 3.5.1 is a bug-fix and maintenance release. It corrects a silent failure in `mwan3rtmon` where route replication to per-interface routing tables was completely non-functional, adds nft set flag-change detection on reload so that changing a set's timeout, counter, or size options takes effect immediately without requiring a full service restart, suppresses spurious stderr noise from ip rule and ip route operations during upgrades and teardown, and removes version number references from comments. @@ -2969,7 +3511,7 @@ Four locations in `mwan3.sh` produced noise on stderr during package upgrades an --- -### 18.14 Version 3.5 +### 18.17 Version 3.5 **Summary:** Version 3.5 is a major architectural release that moves mwan3 out of `table inet fw4` and into its own `table inet mwan3`, eliminating the fw4 rebuild scaffold and the mwan3evtd debounce daemon entirely. @@ -3161,7 +3703,7 @@ mwan3 now renders port ranges using `x-y` (nft native format); the colon separat --- -### 18.15 Version 3.4.1 (Unreleased) +### 18.18 Version 3.4.1 (Unreleased) **Summary:** Builds the per-interface `mwan3_iface_in_*` chains before `mwan3_set_general_nft()` activates `mwan3_prerouting` to avoid a race condition that leads to a wrong interface mark being assigned. Fixes bugs in the `nft list chains` syntax in `stop_service()` and a grep expression that was causing a too-broad match and resulting in traffic for interface `wan` bypassing mwan3 marking. @@ -3203,7 +3745,7 @@ Also removed the flush of `mwan3_postrouting` from `mwan3_set_general_nft`. That --- -### 18.16 Version 3.4 +### 18.19 Version 3.4 **Summary:** Version 3.4 introduces mwan3evtd, a generalised ucode debounce daemon that coalesces rapid-fire events - such as simultaneous interface flaps triggering multiple fw4 reloads - into a single handler execution after the activity settles. This prevents the repeated dnsmasq SIGHUPs that previously caused cache thrash and, in tight-timing scenarios, dnsmasq crashes during concurrent startup. @@ -3242,7 +3784,7 @@ Shell injection in the handler fire path is prevented by passing the command thr --- -### 18.17 Version 3.3.5 +### 18.20 Version 3.3.5 **Summary:** Version 3.3.5 is a single-fix release that suppresses the per-deleted-entry output that `conntrack -D` writes to stdout, which was previously appearing on the console whenever mwan3 start or an fw4 reload triggered the zero-mark conntrack flush. @@ -3256,7 +3798,7 @@ Fix: redirect stdout to `/dev/null` alongside stderr. --- -### 18.18 Version 3.3.4 +### 18.21 Version 3.3.4 **Summary:** Version 3.3.4 closes a class of misrouting bugs caused by the brief window between fw4 flushing `table inet fw4` and mwan3 completing its nft rebuild. Connections established during that window acquire `ct mark=0`; the new `mwan3_flush_stale_conntrack` helper removes all zero-mark conntrack entries after every rebuild and restart, preventing WireGuard persistent-keepalive and similar long-lived UDP from locking in a bad entry indefinitely. A double-rebuild race in `mwan3-fw-rebuild.sh` is also fixed by acquiring the procd lock before checking for empty chains. @@ -3287,7 +3829,7 @@ Fix by acquiring `procd_lock` first and re-checking under the lock, so only one --- -### 18.19 Version 3.3.3 +### 18.22 Version 3.3.3 **Summary:** Version 3.3.3 is a broad bug-fix release addressing several correctness issues: DNAT reply routing was broken by a misplaced `fib daddr type local return` rule that fired before DNAT translation, causing replies to exit via a randomly load-balanced interface; IPv6 ip rules were silently leaked on ifdown because `delete_iface_rules` queried the IPv4 rule table; `mwan3_dnsmasq_hup` never sent SIGHUP because `json_get_var` stores booleans as integers not strings; and the numgen counter was contaminated by inbound and reply traffic. Additional fixes cover a grep substring false-positive in iface chain wiring, unquoted regex variables, a dead function stub, a duplicate function, and missing `mwan3_postrouting` in the stop_service chain lists. A new `bypass_network` UCI option populates the dynamic bypass sets from config, and `mwan3-lb-test` gains fw4 reload detection. @@ -3377,7 +3919,7 @@ Improve the `rt_table_lookup` field: rename label from "Routing table lookup" to --- -### 18.20 Version 3.3.2 +### 18.23 Version 3.3.2 **Summary:** Version 3.3.2 fixes a spurious tracked-IP entry in ubus status output caused by a naming collision between mwan3track's temporary output file and the `TRACK_*` glob used by rpcd. The `mwan3-lb-test` tool gains mandatory client isolation, a Windows test command, and a stale-artifact cleanup subcommand. LuCI receives cross-field family consistency validation in the rule editor, a fix for false "Present (unexpected)" health badges during interface bring-up, source nftset display in the overview rules column, and source nftset support in the traffic path simulator. @@ -3452,7 +3994,7 @@ Grid display: Source and Destination columns now show the nftset name when no IP --- -### 18.21 Version 3.3.1 +### 18.24 Version 3.3.1 **Summary:** Version 3.3.1 adds source nftset matching (`ipset_src`) as a complement to the existing destination nftset, fixes three distinct numgen counter contamination bugs that caused load-balancing distributions to skew under inbound or reply traffic, sweeps orphaned policy chains that accumulate when policies are removed from UCI without an fw4 reload, and corrects IPv6 ip rule detection in the routing health check. The release also adds `nftset_info` as an rpcd ubus method and introduces the `mwan3-lb-test` CLI tool for verifying load-balancing weight distributions against configured policy members. @@ -3524,7 +4066,7 @@ Fix by querying both `-4` and `-6` rule tables and merging the results, matching --- -### 18.22 Version 3.3 +### 18.25 Version 3.3 **Summary:** Version 3.3 adds three major LuCI diagnostic tools - a traffic path Simulator, a static Configuration analyser, and a live Routing health view - backed by two new rpcd ubus methods (`nftset_members` and `routing_health`). The configuration analyser detects undefined references, orphaned sections, and rule shadowing including correct IPv6 CIDR containment checks. The routing health view colour-codes per-interface ip rule and routing table state against live kernel state. The `apk info` vs `apk list -I` version display bug is also fixed. @@ -3568,7 +4110,7 @@ Also adds `nftset_members` and `routing_health` methods to the rpcd module with --- -### 18.23 Version 3.2.3 +### 18.26 Version 3.2.3 **Summary:** Version 3.2.3 improves tracking status visibility by adding per-IP latency and packet-loss detail to `mwan3 status` output and fixing the `check_quality` display to derive its state from mwan3track's runtime files rather than UCI, so changes to UCI without a restart no longer cause the status page to disagree with what is actually running. Stale gateway `TRACK_*`/`LATENCY_*`/`LOSS_*` files from previous PPPoE sessions are cleaned up on each probe list rebuild. The `luci-app-mwan3` PKG_VERSION scheme is fixed to prevent `apk upgrade` from reverting to the official package, and the GitHub Actions APK rename step is corrected to avoid i18n sub-packages overwriting the main package. @@ -3629,7 +4171,7 @@ When `check_quality=1`, tracker latency/loss sentinel values (`999999ms`, `100%` --- -### 18.24 Version 3.2.2 +### 18.27 Version 3.2.2 **Summary:** Version 3.2.2 fixes two misrouting bugs: duplicate jump rules accumulating from repeated fw4 reload cycles caused iface_in chain deletion to fail with "Resource busy", and the unguarded catchall rule in each `mwan3_iface_in_*` chain was stamping IPv6 packets with the IPv4 interface mark on dual-stack physical devices, breaking QUIC/HTTP3 streams that resumed after conntrack expiry. The gateway IP is moved to the front of the tracking probe list so it is always tested. LuCI receives a visual redesign replacing solid alert cards with bordered flex cards, and adds latency and packet-loss columns to the tracking IP table. @@ -3691,7 +4233,7 @@ When `check_quality` is disabled (the default), the columns display "Not enabled --- -### 18.25 Version 3.2.1 +### 18.28 Version 3.2.1 **Summary:** Version 3.2.1 fixes policy status reporting to include all members with their live traffic share percentages (not just the currently-routing member), replaces `killall -HUP dnsmasq` with a procd-aware targeted SIGHUP to avoid crashing instances still in the startup phase, adds the installed mwan3 package version to `mwan3 internal` output, and redesigns the LuCI status pages with structured collapsible sections and an IPv6 troubleshooting pane. LuCI also exposes the `snat6` IPv6 SNAT option on interface configuration. @@ -3743,7 +4285,7 @@ Adds an "IPv6 SNAT" form field to the interface configuration modal, visible onl --- -### 18.26 Version 3.2 +### 18.29 Version 3.2 **Summary:** Version 3.2 adds two significant features. First, opt-in per-interface IPv6 SNAT via the `snat6` UCI option, which corrects BCP38/uRPF drops for router-originated traffic rerouted by `mwan3_output` onto a different WAN than the kernel initially selected at `sendto()`. Second, non-destructive vmap-dispatch mark save/restore: 126 per-mark OR-immediate setter chains replace the previous unmasked connmark operations, making mwan3 fully order-independent with respect to pbr and other fwmark-using packages without requiring coordinated chain priority ordering. @@ -3769,7 +4311,7 @@ The same vmap-dispatch primitive is reused by `mwan3_create_policies_nft` for lo --- -### 18.27 Version 3.1.4 +### 18.30 Version 3.1.4 **Summary:** Version 3.1.4 fixes interoperability with pbr by moving mwan3's prerouting and output chains from priority `mangle + 1` to `mangle - 1`, so mwan3 restores and saves its ct mark bits before pbr injects its own marks at `mangle` priority. With the previous ordering pbr's marks were zeroed before the routing decision and its ip rules never matched. @@ -3787,7 +4329,7 @@ Add `postinst` migration to flush and delete the old chains on upgrade, since nf --- -### 18.28 Version 3.1.3 +### 18.31 Version 3.1.3 **Summary:** Version 3.1.3 fixes three status and policy rendering bugs: single-member policies were emitting spurious "unreachable" entries because the empty-string guard on `mwan3_mark_to_name` never matched; mixed IPv4/IPv6 policies lost one family's members because both shared a single reset list; and equal-weight load-balancing entries were invisible in `mwan3 status` because nft normalises single-element numgen ranges to plain values that the reporting regex did not match. @@ -3817,7 +4359,7 @@ Handle both `N-M : 0xMARK` (weight>1, range preserved by nft) and `N : 0xMARK` ( --- -### 18.29 Version 3.1.2 +### 18.32 Version 3.1.2 **Summary:** Version 3.1.2 improves mwan3rtmon with two fixes: an in-memory route cache replaces the per-event `RTM_GETROUTE` dump for O(1) ECMP path checks, and a ucode-mod-rtnl double-destructor bug that caused a reliable segfault on clean shutdown is eliminated by letting the GC collect the route listener rather than calling `close()` explicitly. @@ -3840,7 +4382,7 @@ Fix: omit the explicit `close()` call and let the GC collect the listener natura --- -### 18.30 Version 3.1.1 +### 18.33 Version 3.1.1 **Summary:** Version 3.1.1 is a broad mwan3track hardening release: the disconnecting threshold is raised to suppress false alarms from single transient ping losses, an exclusive flock prevents ghost duplicate tracker processes per interface, `sockopt_wrap` replaces `exit()` with graceful error returns so a stale source IP or disappearing interface does not abruptly terminate the tracked process, per-host failure logs are suppressed when the reliability threshold is still met, and interface events are processed at the top of the main loop before pinging to avoid a spurious disconnecting state on wakeup from disabled. LuCI adds a track_gateway checkbox to the interface modal and clarifies the flush_conntrack help text. diff --git a/mwan3-nft/files/etc/config/mwan3 b/mwan3-nft/files/etc/config/mwan3 index 517c18ab9..3466c7bf7 100644 --- a/mwan3-nft/files/etc/config/mwan3 +++ b/mwan3-nft/files/etc/config/mwan3 @@ -1,5 +1,5 @@ # For full documentation of mwan3 configuration: -# https://openwrt.org/docs/guide-user/network/wan/multiwan/mwan3#mwan3_configuration +# https://openwrt.org/docs/guide-user/network/wan/multiwan/mwan3-nft#configuration config globals 'globals' option mmx_mask '0x3F00' diff --git a/mwan3-nft/files/etc/hotplug.d/iface/25-mwan3 b/mwan3-nft/files/etc/hotplug.d/iface/25-mwan3 index 33154e0fc..b1e0d4720 100644 --- a/mwan3-nft/files/etc/hotplug.d/iface/25-mwan3 +++ b/mwan3-nft/files/etc/hotplug.d/iface/25-mwan3 @@ -52,9 +52,24 @@ LOG notice "Execute $ACTION event on interface $INTERFACE (${DEVICE:-unknown})" case "$ACTION" in connected) + # A tracker recovery must restore the member's routing before the policy + # re-selects it, so the member is routable the instant it is marked, then + # snap new flows off the worse-metric backups back toward it. The policy + # rebuild commits as one nft transaction so no packet classifies against a + # half-rebuilt policy chain. The snap-back stays outside that batch on + # purpose: a recovery's worse-metric pins point at backups that are still + # up, so a source re-pinned in the window before the snap reaches a live + # member, not a dead one, unlike a teardown whose pins must be flushed in + # the same transaction because they point at the departing member. + mwan3_set_iface_hotplug_state $INTERFACE "online" mwan3_create_iface_nft $INTERFACE $DEVICE + mwan3_create_iface_rules $INTERFACE $DEVICE + mwan3_create_iface_route $INTERFACE $DEVICE + mwan3_nft_batch_start mwan3_set_policies_nft + mwan3_nft_batch_commit + mwan3_flush_worse_metric_sticky $INTERFACE mwan3_flush_unreplied_conntrack ;; ifup) @@ -66,24 +81,52 @@ case "$ACTION" in if [ "$MWAN3_STARTUP" != "init" ]; then mwan3_set_general_rules if [ "$status" = "online" ]; then + + # Same atomic policy rebuild and out-of-batch snap-back as the + # connected arm above; see that comment for why the snap-back is + # not folded into the batch. The batch sits inside the online + # guard so the init path never opens one. + + mwan3_nft_batch_start mwan3_set_policies_nft + mwan3_nft_batch_commit + mwan3_flush_worse_metric_sticky $INTERFACE mwan3_flush_unreplied_conntrack fi fi [ "$ACTION" = ifup ] && procd_running mwan3 "track_$INTERFACE" && procd_send_signal mwan3 "track_$INTERFACE" USR2 ;; disconnected) + # A soft tracker failure drops the member from the policy; flush its + # sticky pins after the rebuild so a pinned source re-evaluates to the + # survivor, the same treatment a hard ifdown gets. The policy rebuild + # and the sticky flush commit as one nft transaction so no packet + # classifies against a half-rebuilt policy. The member's marked flows + # are cleared by the conntrack flush at the handler tail, which is + # kernel state outside nftables and stays a separate operation. + mwan3_set_iface_hotplug_state $INTERFACE "offline" + mwan3_nft_batch_start mwan3_set_policies_nft + mwan3_delete_iface_map_entries $INTERFACE + mwan3_nft_batch_commit ;; ifdown) + # Drop the member from the policy before tearing down its routing, so the + # policy never marks for a member whose rules are already gone; flush its + # sticky sets after the rebuild so the rebuild cannot re-seed them. The + # policy rebuild and the sticky flush commit as one nft transaction; the + # routing teardown and the conntrack flush follow as separate steps. + mwan3_set_iface_hotplug_state $INTERFACE "offline" + mwan3_nft_batch_start + mwan3_set_policies_nft mwan3_delete_iface_map_entries $INTERFACE + mwan3_nft_batch_commit mwan3_delete_iface_rules $INTERFACE mwan3_delete_iface_route $INTERFACE mwan3_delete_iface_nft $INTERFACE procd_running mwan3 "track_$INTERFACE" && procd_send_signal mwan3 "track_$INTERFACE" USR1 - mwan3_set_policies_nft ;; esac diff --git a/mwan3-nft/files/etc/init.d/mwan3 b/mwan3-nft/files/etc/init.d/mwan3 index bc602366d..a930bb94b 100755 --- a/mwan3-nft/files/etc/init.d/mwan3 +++ b/mwan3-nft/files/etc/init.d/mwan3 @@ -66,6 +66,7 @@ start_service() { mwan3_set_connected_sets mwan3_set_custom_sets mwan3_set_general_rules + mwan3_set_member_backstops config_foreach mwan3_ifup interface "init" wait $hotplug_pids mwan3_set_general_nft @@ -134,11 +135,13 @@ stop_service() { # Delete ip rules created by this mwan3 instance. A rule must pass # both gates before it is deleted: - # priority gate -- the priority lies in one of mwan3's three - # configured rule-base ranges (iif, fwmark, unreachable). The + # priority gate -- the priority lies in one of mwan3's four + # rule-base ranges (iif, fwmark, unreachable, source). The # fwmark range extends to fwmark_base + MM_UNREACHABLE to cover # the global blackhole and unreachable rules at fwmark_base + - # MM_BLACKHOLE / MM_UNREACHABLE. + # MM_BLACKHOLE / MM_UNREACHABLE. The source band is not + # configurable; it is derived from the unreachable base and + # sits immediately above the unreachable band. # content gate -- the line either references a mwan3-owned routing # table (lookup with N in 1..MWAN3_INTERFACE_MAX) or carries # an fwmark masked by mwan3's MMX_MASK. The priority gate alone @@ -149,13 +152,15 @@ stop_service() { # /var/run/mwan3/mmx_mask and /var/state/mwan3 (set at start_service # time), so they reflect what the running instance created, not # whatever is currently in /etc/config/mwan3. - local line pref tbl iif_min iif_max fw_min fw_max ur_min ur_max mask + local line pref tbl iif_min iif_max fw_min fw_max ur_min ur_max src_min src_max mask iif_min=$((MWAN3_IIF_RULE_BASE + 1)) iif_max=$((MWAN3_IIF_RULE_BASE + MWAN3_INTERFACE_MAX)) fw_min=$((MWAN3_FWMARK_RULE_BASE + 1)) fw_max=$((MWAN3_FWMARK_RULE_BASE + MM_UNREACHABLE)) ur_min=$((MWAN3_UNREACHABLE_RULE_BASE + 1)) ur_max=$((MWAN3_UNREACHABLE_RULE_BASE + MWAN3_INTERFACE_MAX)) + src_min=$((MWAN3_UNREACHABLE_RULE_BASE + MWAN3_INTERFACE_MAX + 2)) + src_max=$((MWAN3_UNREACHABLE_RULE_BASE + 2 * MWAN3_INTERFACE_MAX + 1)) # ip rule list prints fwmark masks lowercase; normalise for matching mask=$(echo "$MMX_MASK" | tr 'A-Z' 'a-z') $IP rule list | while read -r line; do @@ -166,7 +171,8 @@ stop_service() { # Priority gate if ! { { [ "$pref" -ge "$iif_min" ] && [ "$pref" -le "$iif_max" ]; } || \ { [ "$pref" -ge "$fw_min" ] && [ "$pref" -le "$fw_max" ]; } || \ - { [ "$pref" -ge "$ur_min" ] && [ "$pref" -le "$ur_max" ]; }; }; then + { [ "$pref" -ge "$ur_min" ] && [ "$pref" -le "$ur_max" ]; } || \ + { [ "$pref" -ge "$src_min" ] && [ "$pref" -le "$src_max" ]; }; }; then continue fi # Content gate (a): carries an fwmark masked by mwan3's MMX_MASK @@ -190,40 +196,9 @@ stop_service() { done done - # Flush all mwan3 chains (rules removed, chains stay as skeletons) - for chain_name in $($NFT list chains inet 2>/dev/null | grep "chain mwan3_" | awk '{print $2}'); do - $NFT flush chain inet mwan3 "$chain_name" >/dev/null 2>&1 - done - - # Delete dynamic per-interface and per-policy chains (not the skeleton ones from 10-mwan3.nft) - for chain_name in $($NFT list chains inet 2>/dev/null | grep "chain mwan3_" | awk '{print $2}'); do - case "$chain_name" in - mwan3_prerouting|mwan3_output|mwan3_postrouting|mwan3_ifaces_in|mwan3_rules|mwan3_connected|mwan3_custom|mwan3_dynamic) - # These are skeleton chains from the static .nft file; keep them - ;; - *) - $NFT delete chain inet mwan3 "$chain_name" >/dev/null 2>&1 - ;; - esac - done - - # Final safety flush of skeleton chains (handles any transient failures above) - for chain_name in mwan3_prerouting mwan3_output mwan3_postrouting mwan3_ifaces_in mwan3_rules mwan3_connected mwan3_custom mwan3_dynamic; do - $NFT flush chain inet mwan3 "$chain_name" >/dev/null 2>&1 - done - - # Flush all mwan3 sets - for setname in $($NFT list sets inet mwan3 2>/dev/null | grep "set mwan3_" | awk '{print $2}'); do - $NFT flush set inet mwan3 "$setname" >/dev/null 2>&1 - done - - # Delete dynamic maps (sticky maps) - for mapname in $($NFT list maps inet mwan3 2>/dev/null | grep "map mwan3_" | awk '{print $2}'); do - $NFT flush map inet mwan3 "$mapname" >/dev/null 2>&1 - $NFT delete map inet mwan3 "$mapname" >/dev/null 2>&1 - done - - # Remove the table entirely so a stopped mwan3 leaves no nft residue + # Remove the table entirely. The single atomic delete drops every + # chain, set and map mwan3 created in one kernel transaction, so a + # stopped mwan3 leaves no nft residue. $NFT delete table inet mwan3 >/dev/null 2>&1 rm -rf $MWAN3_STATUS_DIR $MWAN3TRACK_STATUS_DIR @@ -233,6 +208,23 @@ stop_service() { reload_service() { mwan3_init + # The nft table is created only by the skeleton load in start_service, + # which is skipped when the service starts with every interface + # disabled. In that state the table is absent while the status + # directory (created by mwan3_init) exists, so the service reports + # running and a config change drives a reload, not a start. Rebuilding + # the ruleset here would batch every add against a missing table, the + # atomic commit would fail, and this function would return at the commit + # check before the tracker-mismatch restart below. Recover with a full + # stop and start, which reloads the skeleton and recreates the table. + + if ! $NFT list table inet mwan3 >/dev/null 2>&1; then + LOG notice "reload escalated to restart as table inet mwan3 not initialised" + stop + start + return + fi + # Atomically rebuild the entire mwan3 nft ruleset in a single kernel # transaction. mwan3_nft_reload_start opens the outermost batch level # (MWAN3_BATCH_DEPTH 0->1) so all subsequent batch_start/commit calls @@ -256,6 +248,7 @@ reload_service() { # update them after the nft commit so the table is already consistent. mwan3_update_iface_to_table mwan3_set_general_rules + mwan3_set_member_backstops config_foreach mwan3_rebuild_iface_rules interface # Signal mwan3rtmon to reload UCI config and re-sync the custom sets. diff --git a/mwan3-nft/files/etc/mwan3.user b/mwan3-nft/files/etc/mwan3.user index 2e8e51748..ea41becf0 100644 --- a/mwan3-nft/files/etc/mwan3.user +++ b/mwan3-nft/files/etc/mwan3.user @@ -16,5 +16,5 @@ # $DEVICE Physical device name of the interface the action relates to (e.g. "eth0" or "wwan0"). # Note: On an ifdown event, $DEVICE is not available, use $INTERFACE instead. # -# Further documentation can be found here: -# https://openwrt.org/docs/guide-user/network/wan/multiwan/mwan3#alertsnotifications +# Further documentation can be found at: +# https://openwrt.org/docs/guide-user/network/wan/multiwan/mwan3-nft#hotplug_events_mwan3user diff --git a/mwan3-nft/files/lib/mwan3/mwan3-create-iface-route.uc b/mwan3-nft/files/lib/mwan3/mwan3-create-iface-route.uc index de518cc6f..cbb769544 100644 --- a/mwan3-nft/files/lib/mwan3/mwan3-create-iface-route.uc +++ b/mwan3-nft/files/lib/mwan3/mwan3-create-iface-route.uc @@ -4,6 +4,7 @@ import * as rtnl from "rtnl"; import * as uci from "uci"; import * as ubus from "ubus"; +import { log_open, log_msg } from 'mwan3.common'; const RTM_GETROUTE = rtnl.const.RTM_GETROUTE; const RTM_NEWROUTE = rtnl.const.RTM_NEWROUTE; @@ -56,12 +57,14 @@ let tid = 0; cur.foreach("mwan3", "interface", function(s) { tid++; let fam = s.family ?? "ipv4"; - let enabled = +(s.enabled ?? "1"); + let enabled = +(s.enabled ?? "0"); if (enabled && fam == family_name) name_tid[s[".name"]] = tid; }); cur.unload("mwan3"); +log_open("mwan3-create-iface-route"); + let dev_table_map = {}; let uconn = ubus.connect(); if (uconn) { @@ -117,5 +120,5 @@ for (let route in source_routes) { rtnl.request(RTM_NEWROUTE, NLM_F_CREATE | NLM_F_REPLACE, r); let err = rtnl.error(); if (err) - warn(sprintf("mwan3-create-iface-route: table %d: %s\n", table_id, err)); + log_msg("err", sprintf("table %d: %s", table_id, err)); } diff --git a/mwan3-nft/files/lib/mwan3/mwan3-list-routes.uc b/mwan3-nft/files/lib/mwan3/mwan3-list-routes.uc index 98b2a6e94..10931687f 100644 --- a/mwan3-nft/files/lib/mwan3/mwan3-list-routes.uc +++ b/mwan3-nft/files/lib/mwan3/mwan3-list-routes.uc @@ -2,6 +2,7 @@ 'use strict'; import * as rtnl from "rtnl"; +import { is_default_equivalent } from 'mwan3.common'; const RTM_GETROUTE = rtnl.const.RTM_GETROUTE; const NLM_F_DUMP = rtnl.const.NLM_F_DUMP; @@ -18,12 +19,6 @@ function is_cidr_route(route, family_num) { return (family_num == AF_INET) ? (prefix_len < 32) : (prefix_len < 128); } -function is_default_route(route) { - return (route.dst == null || - route.dst == "0.0.0.0/0" || - route.dst == "::/0"); -} - function is_linklocal_route(route) { return (route.dst != null && (match(route.dst, /^fe80::\//) != null || @@ -40,7 +35,7 @@ let seen = {}; for (let route in routes) { if (route.table != table_num) continue; if (!is_cidr_route(route, family_num)) continue; - if (is_default_route(route)) continue; + if (is_default_equivalent(route)) continue; if (is_linklocal_route(route)) continue; if (family_num == AF_INET) { let first_octet = +split(route.dst, ".")[0]; diff --git a/mwan3-nft/files/lib/mwan3/mwan3-manage-rules.uc b/mwan3-nft/files/lib/mwan3/mwan3-manage-rules.uc index a8e634749..81420cefd 100644 --- a/mwan3-nft/files/lib/mwan3/mwan3-manage-rules.uc +++ b/mwan3-nft/files/lib/mwan3/mwan3-manage-rules.uc @@ -2,11 +2,13 @@ 'use strict'; import * as rtnl from "rtnl"; +import { log_open, log_msg } from 'mwan3.common'; const RTM_GETRULE = rtnl.const.RTM_GETRULE; const RTM_DELRULE = rtnl.const.RTM_DELRULE; const RTM_NEWRULE = rtnl.const.RTM_NEWRULE; const RTM_GETROUTE = rtnl.const.RTM_GETROUTE; +const RTM_GETADDR = rtnl.const.RTM_GETADDR; const NLM_F_DUMP = rtnl.const.NLM_F_DUMP; const NLM_F_CREATE = rtnl.const.NLM_F_CREATE; const AF_INET = rtnl.const.AF_INET; @@ -22,6 +24,8 @@ function is_default_route(route) { route.dst == "::/0"); } +log_open("mwan3-manage-rules"); + let mode = ARGV[0]; if (mode == "check") { @@ -59,20 +63,33 @@ if (mode == "check") { let id = +ARGV[1]; let iif_base = +ARGV[2]; let fwmark_base = +ARGV[3]; - let unreachable_base = +ARGV[4]; - let mmx_mask = +ARGV[5]; + let mmx_mask = +ARGV[4]; + let src_prio = +ARGV[5]; let iif_prio = iif_base + id; let fwmark_prio = fwmark_base + id; - let unreachable_prio = unreachable_base + id; for (let family in [AF_INET, AF_INET6]) { let rules = rtnl.request(RTM_GETRULE, NLM_F_DUMP, { family: family }) ?? []; - let fwmark_val = null; for (let rule in rules) { if (rule.table != id) continue; if (rule.action != FR_ACT_TO_TBL) continue; + + if (rule.src != null && rule.priority == src_prio) { + rtnl.request(RTM_DELRULE, 0, { + family: family, + priority: rule.priority, + src: rule.src, + table: rule.table, + action: rule.action + }); + let err = rtnl.error(); + if (err) + log_msg("err", sprintf("delete src rule failed: %s", err)); + continue; + } + if (rule.src != null || rule.dst != null) continue; if (rule.fwmark == null && rule.priority == iif_prio) { @@ -84,9 +101,8 @@ if (mode == "check") { }); let err = rtnl.error(); if (err) - warn(sprintf("mwan3-manage-rules: delete iif rule failed: %s\n", err)); + log_msg("err", sprintf("delete iif rule failed: %s", err)); } else if (rule.fwmark != null && rule.priority == fwmark_prio && rule.fwmask == mmx_mask) { - fwmark_val = rule.fwmark; rtnl.request(RTM_DELRULE, 0, { family: family, priority: rule.priority, @@ -97,27 +113,7 @@ if (mode == "check") { }); let err = rtnl.error(); if (err) - warn(sprintf("mwan3-manage-rules: delete fwmark rule failed: %s\n", err)); - } - } - - if (fwmark_val != null) { - for (let rule in rules) { - if (rule.fwmark != fwmark_val) continue; - if (rule.action != FR_ACT_UNREACHABLE) continue; - if (rule.priority != unreachable_prio) continue; - if (rule.fwmask != mmx_mask) continue; - if (rule.src != null || rule.dst != null) continue; - rtnl.request(RTM_DELRULE, 0, { - family: family, - priority: rule.priority, - fwmark: rule.fwmark, - fwmask: rule.fwmask, - action: rule.action - }); - let err = rtnl.error(); - if (err) - warn(sprintf("mwan3-manage-rules: delete unreachable rule failed: %s\n", err)); + log_msg("err", sprintf("delete fwmark rule failed: %s", err)); } } } @@ -144,7 +140,7 @@ if (mode == "check") { }); let err = rtnl.error(); if (err) - warn(sprintf("mwan3-manage-rules: add blackhole rule failed: %s\n", err)); + log_msg("err", sprintf("add blackhole rule failed: %s", err)); } if (!existing[ur_prio]) { @@ -157,6 +153,55 @@ if (mode == "check") { }); let err = rtnl.error(); if (err) - warn(sprintf("mwan3-manage-rules: add unreachable rule failed: %s\n", err)); + log_msg("err", sprintf("add unreachable rule failed: %s", err)); + } + +} else if (mode == "add-backstop") { + let family_num = (ARGV[1] == "6") ? AF_INET6 : AF_INET; + let prio = +ARGV[2], mark = +ARGV[3]; + let mask = +ARGV[4]; + + let rules = rtnl.request(RTM_GETRULE, NLM_F_DUMP, { family: family_num }) ?? []; + let existing = {}; + for (let r in rules) + if (r.priority != null) + existing[r.priority] = true; + + if (!existing[prio]) { + rtnl.request(RTM_NEWRULE, NLM_F_CREATE, { + family: family_num, + priority: prio, + fwmark: mark, + fwmask: mask, + action: FR_ACT_UNREACHABLE + }); + let err = rtnl.error(); + if (err) + log_msg("err", sprintf("add backstop rule failed: %s", err)); + } + +} else if (mode == "add-src") { + // IPv4 only + + let device = ARGV[1]; + let table_id = +ARGV[2]; + let prio = +ARGV[3]; + + let addrs = rtnl.request(RTM_GETADDR, NLM_F_DUMP, { family: AF_INET }) ?? []; + for (let a in addrs) { + if (a.dev != device) continue; + if (a.scope != 0) continue; + let addr = split(a.local ?? a.address ?? "", "/")[0]; + if (!addr) continue; + rtnl.request(RTM_NEWRULE, NLM_F_CREATE, { + family: AF_INET, + priority: prio, + src: addr, + table: table_id, + action: FR_ACT_TO_TBL + }); + let err = rtnl.error(); + if (err) + log_msg("err", sprintf("add src rule failed: %s", err)); } } diff --git a/mwan3-nft/files/lib/mwan3/mwan3.sh b/mwan3-nft/files/lib/mwan3/mwan3.sh index 3dedbcd81..9a5db3296 100644 --- a/mwan3-nft/files/lib/mwan3/mwan3.sh +++ b/mwan3-nft/files/lib/mwan3/mwan3.sh @@ -266,7 +266,8 @@ _mwan3_render_one_ipset() set_decl="type ${addr_type}; flags interval" [ "$timeout" -gt 0 ] && set_decl="$set_decl, timeout" - set_decl="$set_decl; auto-merge;" + set_decl="$set_decl;" + [ "$timeout" -eq 0 ] && set_decl="$set_decl auto-merge;" [ "$counters" -eq 1 ] && set_decl="$set_decl counter;" [ "$timeout" -gt 0 ] && set_decl="$set_decl timeout ${timeout}s;" [ "$maxelem" -gt 0 ] && set_decl="$set_decl size ${maxelem};" @@ -291,14 +292,35 @@ _mwan3_render_one_ipset() fi # Collect all elements (inline list + loadfile) before entering batch. + # Every element is validated against the nft set element grammar (IPv4 and + # IPv6 addresses, CIDR prefixes and hyphen ranges) so a crafted value + # cannot close the element braces early and inject nft statements into the + # batch that is later committed with nft -f. Anything carrying a character + # outside that class is dropped. + local elements="" line - _add_entry() { elements="${elements:+$elements, }$1"; } + _add_entry() { + case "$1" in + ""|*[!0-9A-Fa-f:./-]*) + LOG warn "config ipset '$name': dropping invalid entry" + return 0 + ;; + esac + elements="${elements:+$elements, }$1" + } config_list_foreach "$section" entry _add_entry if [ -n "$loadfile" ] && [ -f "$loadfile" ]; then while IFS= read -r line; do line="${line%%#*}" line=$(echo "$line" | xargs 2>/dev/null) - [ -n "$line" ] && elements="${elements:+$elements, }$line" + [ -n "$line" ] || continue + case "$line" in + *[!0-9A-Fa-f:./-]*) + LOG warn "config ipset '$name': dropping invalid loadfile element" + continue + ;; + esac + elements="${elements:+$elements, }$line" done < "$loadfile" fi @@ -466,6 +488,41 @@ mwan3_set_general_rules() "$MMX_MASK" } +# Install a static unreachable ip rule for every configured interface mark, in +# the per-member unreachable priority band. These are created at service start +# and reload and removed only at stop, so a packet carrying a member mark whose +# fwmark rule is currently absent (the member is down, disabled, or briefly +# between a delete and re-add) is rejected instead of leaking to the main +# routing table. One rule per member is required because an fwmark rule matches +# an exact value under the mask, not "any member mark"; the reserved last_resort +# marks are never enumerated here, so last_resort behaviour is unaffected. + +mwan3_set_member_backstops() +{ + mwan3_add_backstop() + { + local id family fam + + config_get family "$1" family ipv4 + mwan3_get_iface_id id "$1" + [ -n "$id" ] || return 0 + + if [ "$family" = "ipv4" ]; then + fam=4 + elif [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then + fam=6 + else + return 0 + fi + + ${MWAN3_MANAGE_RULES} add-backstop "$fam" \ + "$((id+MWAN3_UNREACHABLE_RULE_BASE))" \ + "$(mwan3_id2mask id MMX_MASK)" "$MMX_MASK" + } + + config_foreach mwan3_add_backstop interface +} + mwan3_set_general_nft() { local chain_exists restore_vmap save_vmap all_marks @@ -843,16 +900,112 @@ mwan3_delete_iface_map_entries() # Sticky scheme: one set per (rule, family, iface_id) holding # only saddrs (no value side). Removing an interface invalidates every # such set whose name ends in "_"; we flush rather than delete since - # rule chains may still reference the set name. + # rule chains may still reference the set name. The flush lines are pushed + # into the caller's open nft batch so they commit in the same transaction + # as the policy rebuild; the set enumeration reads committed kernel state + # and so stays outside the batch. Call only within an open nft batch. for setname in $($NFT list sets inet 2>/dev/null | \ awk '$1=="set" && $2 ~ /^mwan3_sticky_v[46]_/ { print $2 }'); do case "$setname" in - *_"$id") $NFT flush set inet mwan3 "$setname" 2>/dev/null ;; + *_"$id") mwan3_nft_push "flush set inet mwan3 $setname" ;; esac done } +mwan3_flush_worse_metric_sticky() +{ + # On recovery of a better-metric member, snap new flows off the worse-metric + # backups it is now preferred over. For each policy the recovered interface + # participates in, flush the per-rule sticky sets of that policy's same-family + # members whose metric is strictly worse, scoped to the rules that use the + # policy, so pinned sources re-evaluate to the recovered member on their next + # new flow while established flows ride their conntrack marks to completion + # undisturbed. The scope is per policy and per rule by construction: each set + # name is built from the rule name and the member id rather than flushing + # every set ending in an id, so a member that is also a deliberate-stickiness + # member of a separate load-balance policy keeps those pins. + + local recovered="$1" recovered_id rec_family rec_fam_short + local _snap_pairs="" + + mwan3_get_iface_id recovered_id "$recovered" + [ -n "$recovered_id" ] || return 0 + + # mwan3 runs an independent metric contest per address family, so only + # same-family members compete with the recovered one; resolve its family once + # and snap only that family's backups. + + config_get rec_family "$recovered" family ipv4 + if [ "$rec_family" = "ipv6" ]; then + rec_fam_short="v6" + else + rec_fam_short="v4" + fi + + # Record one "policy=worse-id" pair per strictly-worse-metric same-family + # member of each policy the recovered interface is in. The recovered + # interface's own id is excluded so its own pins are never cleared. + + _mwan3_snap_collect() { + local policy="$1" rec_metric="" + + # The recovered interface's metric in this policy sets the threshold; + # take the lowest if it appears more than once. + + _mwan3_snap_rec_metric() { + local m_iface m_metric + config_get m_iface "$1" interface + [ "$m_iface" = "$recovered" ] || return + config_get m_metric "$1" metric 1 + { [ -z "$rec_metric" ] || [ "$m_metric" -lt "$rec_metric" ]; } && rec_metric="$m_metric" + } + config_list_foreach "$policy" use_member _mwan3_snap_rec_metric + + # Policy does not include the recovered interface: nothing to snap. + + [ -n "$rec_metric" ] || return + + _mwan3_snap_worse() { + local m_iface m_metric m_id m_family + config_get m_iface "$1" interface + [ -n "$m_iface" ] || return + config_get m_family "$m_iface" family ipv4 + [ "$m_family" = "$rec_family" ] || return + config_get m_metric "$1" metric 1 + [ "$m_metric" -gt "$rec_metric" ] || return + mwan3_get_iface_id m_id "$m_iface" + [ -n "$m_id" ] || return + [ "$m_id" = "$recovered_id" ] && return + case " $_snap_pairs " in + *" ${policy}=${m_id} "*) ;; + *) _snap_pairs="$_snap_pairs ${policy}=${m_id}" ;; + esac + } + config_list_foreach "$policy" use_member _mwan3_snap_worse + } + config_foreach _mwan3_snap_collect policy + + [ -n "$_snap_pairs" ] || return 0 + + # For every rule, flush the worse-metric members' per-rule sticky sets of the + # policy that rule uses, in the recovered member's family. A set name that + # does not exist is a harmless no-op. + + _mwan3_snap_flush() { + local r_policy pair p_name p_id + config_get r_policy "$1" use_policy + [ -n "$r_policy" ] || return + for pair in $_snap_pairs; do + p_name="${pair%=*}" + p_id="${pair##*=}" + [ "$p_name" = "$r_policy" ] || continue + $NFT flush set inet mwan3 "mwan3_sticky_${rec_fam_short}_${1}_${p_id}" 2>/dev/null + done + } + config_foreach _mwan3_snap_flush rule +} + mwan3_create_iface_route() { local family id source_routing _fam_num @@ -904,7 +1057,8 @@ mwan3_create_iface_rules() $IP rule add pref $((id+MWAN3_IIF_RULE_BASE)) iif "$2" lookup "$id" 2>/dev/null $IP rule add pref $((id+MWAN3_FWMARK_RULE_BASE)) fwmark "$(mwan3_id2mask id MMX_MASK)/$MMX_MASK" lookup "$id" 2>/dev/null - $IP rule add pref $((id+MWAN3_UNREACHABLE_RULE_BASE)) fwmark "$(mwan3_id2mask id MMX_MASK)/$MMX_MASK" unreachable 2>/dev/null + [ "$family" = "ipv4" ] && ${MWAN3_MANAGE_RULES} add-src "$2" "$id" \ + "$((id + MWAN3_UNREACHABLE_RULE_BASE + MWAN3_INTERFACE_MAX + 1))" } mwan3_delete_iface_rules() @@ -915,13 +1069,13 @@ mwan3_delete_iface_rules() [ -n "$id" ] || return 0 ${MWAN3_MANAGE_RULES} delete-iface "$id" \ - "$MWAN3_IIF_RULE_BASE" "$MWAN3_FWMARK_RULE_BASE" \ - "$MWAN3_UNREACHABLE_RULE_BASE" "$MMX_MASK" + "$MWAN3_IIF_RULE_BASE" "$MWAN3_FWMARK_RULE_BASE" "$MMX_MASK" \ + "$((id + MWAN3_UNREACHABLE_RULE_BASE + MWAN3_INTERFACE_MAX + 1))" } mwan3_set_policy() { - local id iface family metric weight device is_lowest is_offline + local id iface family metric weight is_lowest is_offline is_lowest=0 config_get iface "$1" interface @@ -929,7 +1083,6 @@ mwan3_set_policy() config_get weight "$1" weight 1 [ -n "$iface" ] || return 0 - network_get_device device "$iface" [ "$metric" -gt $DEFAULT_LOWEST_METRIC ] && LOG warn "Member interface $iface has >$DEFAULT_LOWEST_METRIC metric. Not appending to policy" && return 0 mwan3_get_iface_id id "$iface" @@ -976,30 +1129,28 @@ mwan3_set_policy() if [ $is_offline -eq 0 ]; then - # Accumulate members per family: "iface_name:id:weight" tuples + # Accumulate members per family: "iface_name:id:weight" tuples. + # The branches mirror the metric contest above, so a member only + # accumulates through its interface family's contest: an IPv6 + # member is excluded on a kernel without IPv6, and an + # unrecognised family value is never accumulated. if [ "$family" = "ipv4" ]; then policy_members_v4="$policy_members_v4 $iface:$id:$weight" - else + elif [ "$family" = "ipv6" ] && [ $NO_IPV6 -eq 0 ]; then policy_members_v6="$policy_members_v6 $iface:$id:$weight" fi - elif [ -n "$device" ]; then - - # Offline interface with device: record for fallback out-device rule - - policy_offline_devices="$policy_offline_devices $iface:$device" fi } mwan3_create_policies_nft() { local last_resort lowest_metric_v4 lowest_metric_v6 total_weight_v4 total_weight_v6 - local policy policy_members_v4 policy_members_v6 policy_offline_devices + local policy policy_members_v4 policy_members_v6 policy="$1" policy_members_v4="" policy_members_v6="" - policy_offline_devices="" config_get last_resort "$1" last_resort unreachable @@ -1097,22 +1248,6 @@ mwan3_create_policies_nft() done fi - # Add offline device fallback rules - - local dev_entry offline_iface offline_device - - # Only add if no online members - - if [ "$total_weight" -eq 0 ]; then - for dev_entry in $policy_offline_devices; do - offline_iface="${dev_entry%%:*}" - offline_device="${dev_entry#*:}" - mwan3_nft_exec add rule inet mwan3 "mwan3_policy_$policy" \ - oifname "$offline_device" meta mark \& "$MMX_MASK" == 0 \ - "$(mwan3_nft_mark_expr $MMX_DEFAULT $MMX_MASK)" - done - fi - # Add last resort rule case "$last_resort" in @@ -1214,7 +1349,7 @@ mwan3_set_user_nft_rule() local ipset_name ipset_src family proto policy src_ip src_port src_iface src_dev local sticky dest_ip dest_port use_policy timeout policy local global_logging rule_logging loglevel rule_policy rule ipv - local enabled fwmark fwmask _check_set _set_info + local enabled fwmark fwmask _check_set _set_info _fam_anchored config_get_bool enabled "$1" enabled 1 [ "$enabled" -eq 1 ] || return @@ -1262,18 +1397,30 @@ mwan3_set_user_nft_rule() [ "$family" = "ipv4" ] && [ "$ipv" = "ipv6" ] && return [ "$family" = "ipv6" ] && [ "$ipv" = "ipv4" ] && return - # family=any rules whose nft expression has no IP-version-specific element - # (no src_ip/dest_ip/ipset/ipset_src) generate identical output on both the - # ipv4 and ipv6 passes. Skip the ipv6 pass to avoid pushing a duplicate - # rule. The ipv4 pass output already matches IPv6 traffic at runtime - # because the match operates on family-agnostic fields (meta mark, l4proto, - # port, iifname). Exception: proto=icmp requires both passes because ICMP - # (protocol 1) and ICMPv6 (protocol 58) are distinct L4 protocols. + # A family-anchoring element (src_ip/dest_ip/ipset/ipset_src) ties the + # rendered expression to one address family per pass. This list must + # cover every matcher whose nft form differs by address family; the + # de-duplication guard and the sticky block both key on the flag. + + _fam_anchored=0 + if [ -n "$src_ip" ] || [ -n "$dest_ip" ] || \ + [ -n "$ipset_name" ] || [ -n "$ipset_src" ]; then + _fam_anchored=1 + fi + + # family=any rules with no family-anchoring element render once, on + # the ipv4 pass: the emitted line matches on family-agnostic fields + # (meta mark, l4proto, port, iifname, ether saddr) and carries no + # nfproto guard, so it covers both families at its config position, + # and the sticky block builds the per-rule chain complete for both + # families on that pass. Skip the ipv6 pass to avoid a duplicate + # line. Exception: proto=icmp requires both passes because ICMP + # (protocol 1) and ICMPv6 (protocol 58) are distinct L4 protocols; + # the ipv6 pass then emits its own line but must not rebuild the + # sticky chain. if [ "$family" = "any" ] && [ "$ipv" = "ipv6" ] && \ - [ -z "$src_ip" ] && [ -z "$dest_ip" ] && \ - [ -z "$ipset_name" ] && [ -z "$ipset_src" ] && \ - [ "$proto" != "icmp" ]; then + [ $_fam_anchored -eq 0 ] && [ "$proto" != "icmp" ]; then return fi @@ -1300,7 +1447,13 @@ mwan3_set_user_nft_rule() # the src_ip/dest_ip address validation above: a set of type ipv4_addr # cannot appear in an "ip6 daddr @set" expression, and vice versa. - local _ipset_name_uci_type="" _ipset_src_uci_type="" + local _ipset_name_uci_type="" _ipset_src_uci_type="" _pass_addrtype + if [ "$ipv" = "ipv4" ]; then + _pass_addrtype="ipv4_addr" + else + _pass_addrtype="ipv6_addr" + fi + for _check_set in "$ipset_name" "$ipset_src"; do [ -z "$_check_set" ] && continue local _set_info @@ -1339,6 +1492,25 @@ mwan3_set_user_nft_rule() [ "$ipv" = "ipv6" ] && [ "$_uci_addrtype" = "ipv4_addr" ] && return continue fi + + # An earlier rule in this render may already have pre-created + # the set into the open batch, which the kernel query above + # cannot see. Consult the render's own record instead: the + # first pre-creation establishes the set's type for the whole + # render, and every later reference is treated exactly as if + # the kernel held a set of that type. + + case " $MWAN3_RENDER_SETS " in + *" ${_check_set}=${_pass_addrtype} "*) + continue + ;; + *" ${_check_set}="*) + [ "$family" = "any" ] && return + LOG warn "Rule $rule: set '$_check_set' pre-created as the other address family by an earlier rule, incompatible with family $family" + return + ;; + esac + # Not UCI-managed: apply existing guard for external sets. # For family=any, skip the ipv6 pass; the ipv4 pass will pre-create # the set as ipv4_addr until an external creator establishes its type. @@ -1439,15 +1611,20 @@ mwan3_set_user_nft_rule() # set is missing, which would kill ALL user rules. # UCI-managed sets are already added to the batch by mwan3_render_config_ipsets; # skip the kernel check and pre-creation for those. + # A set this render has already pre-created is in the record and is + # never added twice: two adds of one name in a batch are rejected + # when the types differ, and redundant when they match. if [ -z "$_ipset_name_uci_type" ] && \ ! $NFT list set inet mwan3 "$ipset_name" &>/dev/null; then - LOG notice "Creating missing nft set '$ipset_name' for rule $rule" - if [ "$ipv" = "ipv4" ]; then - mwan3_nft_push "add set inet mwan3 $ipset_name { type ipv4_addr; flags interval; auto-merge; }" - else - mwan3_nft_push "add set inet mwan3 $ipset_name { type ipv6_addr; flags interval; auto-merge; }" - fi + case " $MWAN3_RENDER_SETS " in + *" ${ipset_name}=${_pass_addrtype} "*) ;; + *) + LOG notice "Creating missing nft set '$ipset_name' for rule $rule" + mwan3_nft_push "add set inet mwan3 $ipset_name { type ${_pass_addrtype}; flags interval; auto-merge; }" + MWAN3_RENDER_SETS="$MWAN3_RENDER_SETS ${ipset_name}=${_pass_addrtype}" + ;; + esac fi if [ "$ipv" = "ipv4" ]; then nft_match="$nft_match ip daddr @$ipset_name" @@ -1461,12 +1638,14 @@ mwan3_set_user_nft_rule() if [ -n "$ipset_src" ]; then if [ -z "$_ipset_src_uci_type" ] && \ ! $NFT list set inet mwan3 "$ipset_src" &>/dev/null; then - LOG notice "Creating missing nft set '$ipset_src' for rule $rule" - if [ "$ipv" = "ipv4" ]; then - mwan3_nft_push "add set inet mwan3 $ipset_src { type ipv4_addr; flags interval; auto-merge; }" - else - mwan3_nft_push "add set inet mwan3 $ipset_src { type ipv6_addr; flags interval; auto-merge; }" - fi + case " $MWAN3_RENDER_SETS " in + *" ${ipset_src}=${_pass_addrtype} "*) ;; + *) + LOG notice "Creating missing nft set '$ipset_src' for rule $rule" + mwan3_nft_push "add set inet mwan3 $ipset_src { type ${_pass_addrtype}; flags interval; auto-merge; }" + MWAN3_RENDER_SETS="$MWAN3_RENDER_SETS ${ipset_src}=${_pass_addrtype}" + ;; + esac fi if [ "$ipv" = "ipv4" ]; then nft_match="$nft_match ip saddr @$ipset_src" @@ -1509,7 +1688,7 @@ mwan3_set_user_nft_rule() # like default_rule (family ipv4, no saddr/daddr) generates a bare # "meta mark ... jump policy" that matches IPv6 traffic too. - if [ -z "$src_ip" ] && [ -z "$dest_ip" ] && [ -z "$ipset_name" ] && [ -z "$ipset_src" ]; then + if [ $_fam_anchored -eq 0 ]; then if [ "$family" = "ipv4" ]; then nft_match="${nft_match:+$nft_match }meta nfproto ipv4" elif [ "$family" = "ipv6" ]; then @@ -1549,49 +1728,86 @@ mwan3_set_user_nft_rule() # bit. The save side mirrors this with per-member "update @set" rules # guarded on (meta mark & MMX) == . + # The per-rule chain is shared by every mwan3_rules line that + # jumps to it, so it is built exactly once, complete for every + # family whose traffic can reach it. A family=any rule with no + # family-anchoring element serves both families from the single + # line emitted on the ipv4 pass, so that pass builds both + # families' arms; for proto=icmp the ipv6 pass also reaches this + # block and only points its own line at the finished chain. The + # layout is load-bearing: every lookup precedes the single + # fall-through, so a pinned source ORs its member mark into the + # packet before the fall-through can hand it to the policy, and + # the update rules follow. + local _policy_member_marks _entry _m_id _m_mark _setname - local _fam_short _saddr_kw _addr_type - if [ "$ipv" = "ipv4" ]; then - _fam_short="v4"; _saddr_kw="ip saddr"; _addr_type="ipv4_addr" - else - _fam_short="v6"; _saddr_kw="ip6 saddr"; _addr_type="ipv6_addr" + local _fam_short _saddr_kw _addr_type _sticky_families _sfam + + _sticky_families="$ipv" + if [ "$family" = "any" ] && [ $_fam_anchored -eq 0 ]; then + if [ "$ipv" = "ipv6" ]; then + _sticky_families="" + elif [ $NO_IPV6 -eq 0 ]; then + _sticky_families="ipv4 ipv6" + fi fi - mwan3_get_policy_members_for_family "$use_policy" "$ipv" + _sticky_fam_vars() { + if [ "$1" = "ipv4" ]; then + _fam_short="v4"; _saddr_kw="ip saddr"; _addr_type="ipv4_addr" + else + _fam_short="v6"; _saddr_kw="ip6 saddr"; _addr_type="ipv6_addr" + fi + } - # Create sticky rule chain if it doesn't exist yet. The chain was - # already flushed in the preamble of mwan3_set_user_rules, so both - # ipv4 and ipv6 passes can add their rules without interference. + if [ -n "$_sticky_families" ]; then - mwan3_nft_push "add chain inet mwan3 mwan3_rule_$1" + # Create sticky rule chain if it doesn't exist yet. The chain + # was already flushed in the preamble of mwan3_set_user_rules. - # Per-member sticky sets and lookup rules. + mwan3_nft_push "add chain inet mwan3 mwan3_rule_$1" - for _entry in $_policy_member_marks; do - _m_id="${_entry%%:*}" - _m_mark="${_entry##*:}" - _setname="mwan3_sticky_${_fam_short}_${rule}_${_m_id}" + # Per-member sticky sets and lookup rules, every served + # family's lookups ahead of the fall-through. - $NFT list set inet mwan3 "$_setname" &>/dev/null || \ - mwan3_nft_push "add set inet mwan3 $_setname { type ${_addr_type}; flags timeout; timeout ${timeout}s; }" + for _sfam in $_sticky_families; do + _sticky_fam_vars "$_sfam" + mwan3_get_policy_members_for_family "$use_policy" "$_sfam" - mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 ${_saddr_kw} @${_setname} jump mwan3_or_meta_$(mwan3_or_chain_suffix "$_m_mark")" - done + for _entry in $_policy_member_marks; do + _m_id="${_entry%%:*}" + _m_mark="${_entry##*:}" + _setname="mwan3_sticky_${_fam_short}_${rule}_${_m_id}" - # Fall through to policy for new flows (no sticky entry hit -> mark still 0). + $NFT list set inet mwan3 "$_setname" &>/dev/null || \ + mwan3_nft_push "add set inet mwan3 $_setname { type ${_addr_type}; flags timeout; timeout ${timeout}s; }" - mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 meta mark & $MMX_MASK == 0 jump mwan3_policy_$use_policy" + mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 ${_saddr_kw} @${_setname} jump mwan3_or_meta_$(mwan3_or_chain_suffix "$_m_mark")" + done + done - # After the policy assigns a mark, populate the matching per-member - # sticky set so subsequent packets from this saddr stay on the same WAN. + # Fall through to policy for new flows (no sticky entry hit -> + # mark still 0). - for _entry in $_policy_member_marks; do - _m_id="${_entry%%:*}" - _m_mark="${_entry##*:}" - _setname="mwan3_sticky_${_fam_short}_${rule}_${_m_id}" + mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 meta mark & $MMX_MASK == 0 jump mwan3_policy_$use_policy" - mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 meta mark & $MMX_MASK == $_m_mark update @${_setname} { ${_saddr_kw} timeout ${timeout}s }" - done + # After the policy assigns a mark, populate the matching + # per-member sticky set so subsequent packets from this saddr + # stay on the same WAN. + + for _sfam in $_sticky_families; do + _sticky_fam_vars "$_sfam" + mwan3_get_policy_members_for_family "$use_policy" "$_sfam" + + for _entry in $_policy_member_marks; do + _m_id="${_entry%%:*}" + _m_mark="${_entry##*:}" + _setname="mwan3_sticky_${_fam_short}_${rule}_${_m_id}" + + mwan3_nft_push "add rule inet mwan3 mwan3_rule_$1 meta mark & $MMX_MASK == $_m_mark update @${_setname} { ${_saddr_kw} timeout ${timeout}s }" + done + done + fi policy_action="jump mwan3_rule_$1" fi @@ -1638,14 +1854,21 @@ mwan3_set_user_iface_rules() mwan3_set_user_rules() { - local ipv + # Record of the external sets this render has pre-created into the + # batch, as space-delimited "name=addrtype" tokens, read and appended + # to by mwan3_set_user_nft_rule through dynamic scoping. Kernel + # queries inside an open batch see pre-batch state, so the renderer + # carries its own shadow of what the batch already holds. Reset per + # render, so repeated renders emit identical batches. + + local MWAN3_RENDER_SETS="" mwan3_nft_batch_start mwan3_nft_push "flush chain inet mwan3 mwan3_rules" # Pre-create and flush per-rule chains for all enabled UCI rules, before - # the per-family loop. Sourcing names from UCI (not the kernel) prevents + # the render pass. Sourcing names from UCI (not the kernel) prevents # chains for deleted rules from being recreated after reload. "add chain" # is idempotent: it recreates a chain deleted by mwan3_nft_reload_start or # is a no-op if the chain already exists (hotplug path). @@ -1659,10 +1882,22 @@ mwan3_set_user_rules() } config_foreach _init_rule_chain rule - for ipv in ipv4 ipv6; do - [ "$ipv" = "ipv6" ] && [ $NO_IPV6 -ne 0 ] && continue - config_foreach mwan3_set_user_nft_rule rule "$ipv" - done + # Render the rules in one pass over the config, in config order, + # emitting each rule's IPv4 and IPv6 variants at that rule's position. + # The chain is evaluated first-match, so a rule's position in the chain + # must equal its position in the config; rendering per family instead + # would place every IPv6-specific rule after every IPv4-pass rule, + # letting a family-agnostic rule shadow IPv6 rules the config places + # before it. + + _render_rule() { + local _rr_ipv + for _rr_ipv in ipv4 ipv6; do + [ "$_rr_ipv" = "ipv6" ] && [ $NO_IPV6 -ne 0 ] && continue + mwan3_set_user_nft_rule "$1" "$_rr_ipv" + done + } + config_foreach _render_rule rule mwan3_nft_batch_commit } @@ -2018,12 +2253,13 @@ mwan3_flush_conntrack() config_list_foreach "$interface" flush_conntrack handle_flush "$action" fi - # On ifdown, selectively flush conntrack entries for this interface's mark. - # This forces flows that were using the failed WAN to immediately re-establish - # via the new policy rather than waiting for a TCP retransmit timeout. - # More targeted than the UCI flush_conntrack mechanism which flushes everything. + # On a teardown transition (a hard ifdown or a soft tracker disconnect), + # selectively flush conntrack entries carrying this interface's mark. This + # forces flows that were using the failed WAN to immediately re-establish via + # the new policy rather than waiting for a TCP retransmit timeout. More + # targeted than the UCI flush_conntrack mechanism which flushes everything. - if [ "$action" = "ifdown" ] && [ -e "$CONNTRACK_FILE" ]; then + if { [ "$action" = "ifdown" ] || [ "$action" = "disconnected" ]; } && [ -e "$CONNTRACK_FILE" ]; then local iface_id iface_mark mwan3_get_iface_id iface_id "$interface" if [ -n "$iface_id" ]; then diff --git a/mwan3-nft/files/usr/sbin/mwan3-diag b/mwan3-nft/files/usr/sbin/mwan3-diag index 170ee75f3..fb53c12cf 100644 --- a/mwan3-nft/files/usr/sbin/mwan3-diag +++ b/mwan3-nft/files/usr/sbin/mwan3-diag @@ -87,14 +87,43 @@ let mwan3_cfg = null; let pub6_seen = {}; let pub4_seen = {}; +let lit_alias = {}; + +// Register one public address. The placeholder map is keyed and numbered on +// normalised forms only; when the literal spelling differs from the +// normalised form it is recorded as an alias so free-form text carrying +// that spelling is still replaced. function register(addr_str) { if (!addr_str) return; + let lit = split(addr_str, '/')[0]; let norm = norm_addr(addr_str); if (!norm) return; - let sa = sock.sockaddr(split(addr_str, '/')[0]); + let sa = sock.sockaddr(lit); if (sa.family == 10 && is_public_v6(norm)) pub6_seen[norm] = 1; else if (sa.family == 2 && is_public_v4(norm)) pub4_seen[norm] = 1; + else return; + if (lit != norm) lit_alias[lit] = norm; +} + +// Sweep a block of free-form text for address literals. Splitting on any +// run of characters that cannot occur inside an address or prefix keeps +// every address-bearing token intact; a token that is not a valid public +// address fails validation inside register() and drops out. A colon-less +// token must be a full dotted quad: the address parser also accepts +// shorthand forms like "1.0", which free-form text produces as ordinary +// decimal fragments, and registering one would alias that fragment to a +// placeholder everywhere in the report. + +function scan_text(text) { + if (!text) return; + for (let tok in split(text, /[^0-9a-fA-F:.\/]+/)) { + if (!length(tok)) continue; + if (index(tok, ':') == -1 && + !match(tok, /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+(\/[0-9]+)?$/)) + continue; + register(tok); + } } for (let n in nb_all) { register(n.dst); } @@ -110,57 +139,8 @@ if (mwan3_st?.interfaces) // Register any addresses appearing in the mwan3 UCI config (e.g. track_ip // values not yet reflected in routes/status). -if (mwan3_cfg) - for (let line in split(mwan3_cfg, '\n')) - for (let tok in split(replace(line, "'", ' '), ' ')) - register(trim(tok)); -// --------------------------------------------------------------------------- -// Phase 3: build deterministic placeholder map -// Sort addresses so the same address always gets the same placeholder across -// multiple runs (e.g. v3.4 run vs v3.5.1 run). -// --------------------------------------------------------------------------- - -let pub6_list = sort(keys(pub6_seen)); -let pub4_list = sort(keys(pub4_seen)); -let addr_map = {}; -for (let i = 0; i < length(pub6_list); i++) - addr_map[pub6_list[i]] = sprintf('PUB6_%d', i + 1); -for (let i = 0; i < length(pub4_list); i++) - addr_map[pub4_list[i]] = sprintf('PUB4_%d', i + 1); - -// --------------------------------------------------------------------------- -// Anonymisation helpers -// --------------------------------------------------------------------------- - -// Anonymise a single address string (preserves any /prefix-length suffix). -function anon(addr_str) { - if (!addr_str) return addr_str; - let parts = split(addr_str, '/'); - let norm = norm_addr(parts[0]); - let ph = norm ? (addr_map[norm] ?? null) : null; - if (ph) return ph + (length(parts) > 1 ? '/' + parts[1] : ''); - return addr_str; -} - -// Anonymise a block of free-form text (nft output, log lines, etc.). -// Sorted longest-first so longer addresses are replaced before any shorter -// prefix they might share (prevents partial substitutions). -let _sorted_addrs = sort(keys(addr_map), (a, b) => length(b) - length(a)); -function anon_text(text) { - if (!text) return '(no output)'; - for (let addr in _sorted_addrs) { - let ph = addr_map[addr]; - let pos = 0; - while (true) { - let found = index(text, addr, pos); - if (found == -1) break; - text = substr(text, 0, found) + ph + substr(text, found + length(addr)); - pos = found + length(ph); - } - } - return text; -} +scan_text(mwan3_cfg); // Strip elements from user-defined nft sets, replacing with { ... }. // mwan3 infrastructure sets (names starting with mwan3_) are left intact: @@ -197,13 +177,102 @@ function strip_set_elements(text) { return join('\n', out); } -// Run a shell command and return its anonymised output. -function pcmd(cmd) { +// Run a shell command and return its raw output. +function rcmd(cmd) { let f = fs.popen(cmd); if (!f) return '(failed to run: ' + cmd + ')\n'; let out = f.read('all'); f.close(); - return anon_text(out || '(no output)\n'); + return out || '(no output)\n'; +} + +// --------------------------------------------------------------------------- +// Detect which table mwan3 is using (v3.4 = inet fw4, v3.5+ = inet mwan3) +// --------------------------------------------------------------------------- + +let mwan3_tbl = 'inet mwan3'; +{ + let f = fs.popen('nft list chain inet mwan3 mwan3_prerouting 2>/dev/null | head -1'); + let l = f?.read('line'); + f?.close(); + if (!l || !length(trim(l))) mwan3_tbl = 'inet fw4'; +} + +// --------------------------------------------------------------------------- +// Capture the free-form text sections the report prints and sweep them for +// address literals. Some public addresses exist only in this text, e.g. NAT +// map elements derived from no route or interface address, or logged +// addresses no longer present in live state, and are registered here so +// they receive placeholders. Each section is captured exactly once and the +// captured text itself is printed later, so no literal can appear in the +// printed copy without having been swept. The mwan3 table is stripped of +// user-set elements before the sweep so redacted addresses are not +// assigned placeholders. +// --------------------------------------------------------------------------- + +let nft_mwan3_txt = strip_set_elements(rcmd(sprintf('nft list table %s 2>/dev/null', mwan3_tbl))); +let fw4_mangle_pre_txt = rcmd('nft list chain inet fw4 mangle_prerouting 2>/dev/null'); +let fw4_mangle_out_txt = rcmd('nft list chain inet fw4 mangle_output 2>/dev/null'); +let fw4_output_txt = rcmd('nft list chain inet fw4 output 2>/dev/null'); +let log_txt = rcmd('logread 2>/dev/null | grep -E "mwan3track\\[|mwan3-hotplug\\[|mwan3-init\\[|mwan3\\[|mwan3rtmon\\[|mwan3-manage-rules\\[|mwan3-create-iface-route\\[" | tail -200'); + +scan_text(nft_mwan3_txt); +scan_text(fw4_mangle_pre_txt); +scan_text(fw4_mangle_out_txt); +scan_text(fw4_output_txt); +scan_text(log_txt); + +// --------------------------------------------------------------------------- +// Phase 3: build deterministic placeholder map +// Sort addresses so the same address always gets the same placeholder across +// multiple runs (e.g. v3.4 run vs v3.5.1 run). +// --------------------------------------------------------------------------- + +let pub6_list = sort(keys(pub6_seen)); +let pub4_list = sort(keys(pub4_seen)); +let addr_map = {}; +for (let i = 0; i < length(pub6_list); i++) + addr_map[pub6_list[i]] = sprintf('PUB6_%d', i + 1); +for (let i = 0; i < length(pub4_list); i++) + addr_map[pub4_list[i]] = sprintf('PUB4_%d', i + 1); + +// A literal spelling that normalises to a registered address receives the +// registered form's placeholder, so both spellings are replaced in text. + +for (let lit, norm in lit_alias) + if (addr_map[norm]) addr_map[lit] = addr_map[norm]; + +// --------------------------------------------------------------------------- +// Anonymisation helpers +// --------------------------------------------------------------------------- + +// Anonymise a single address string (preserves any /prefix-length suffix). +function anon(addr_str) { + if (!addr_str) return addr_str; + let parts = split(addr_str, '/'); + let norm = norm_addr(parts[0]); + let ph = norm ? (addr_map[norm] ?? null) : null; + if (ph) return ph + (length(parts) > 1 ? '/' + parts[1] : ''); + return addr_str; +} + +// Anonymise a block of free-form text (nft output, log lines, etc.). +// Sorted longest-first so longer addresses are replaced before any shorter +// prefix they might share (prevents partial substitutions). +let _sorted_addrs = sort(keys(addr_map), (a, b) => length(b) - length(a)); +function anon_text(text) { + if (!text) return '(no output)'; + for (let addr in _sorted_addrs) { + let ph = addr_map[addr]; + let pos = 0; + while (true) { + let found = index(text, addr, pos); + if (found == -1) break; + text = substr(text, 0, found) + ph + substr(text, found + length(addr)); + pos = found + length(ph); + } + } + return text; } // --------------------------------------------------------------------------- @@ -224,18 +293,6 @@ function tbl_label(n) { return name ? sprintf('%s (%d)', name, n) : sprintf('%d', n); } -// --------------------------------------------------------------------------- -// Detect which table mwan3 is using (v3.4 = inet fw4, v3.5+ = inet mwan3) -// --------------------------------------------------------------------------- - -let mwan3_tbl = 'inet mwan3'; -{ - let f = fs.popen('nft list chain inet mwan3 mwan3_prerouting 2>/dev/null | head -1'); - let l = f?.read('line'); - f?.close(); - if (!l || !length(trim(l))) mwan3_tbl = 'inet fw4'; -} - // --------------------------------------------------------------------------- // mwan3 version // --------------------------------------------------------------------------- @@ -407,20 +464,20 @@ else // --- full mwan3 nft table --- printf('=== mwan3 nft table (%s) ===\n', mwan3_tbl); -printf('%s\n', strip_set_elements(pcmd(sprintf('nft list table %s 2>/dev/null', mwan3_tbl)))); +printf('%s\n', anon_text(nft_mwan3_txt)); // --- fw4 mangle chains (relevant to mwan3 mark interactions) --- printf('=== fw4 mangle_prerouting chain ===\n'); -printf('%s\n', pcmd('nft list chain inet fw4 mangle_prerouting 2>/dev/null')); +printf('%s\n', anon_text(fw4_mangle_pre_txt)); printf('=== fw4 mangle_output chain ===\n'); -printf('%s\n', pcmd('nft list chain inet fw4 mangle_output 2>/dev/null')); +printf('%s\n', anon_text(fw4_mangle_out_txt)); printf('=== fw4 output chain ===\n'); -printf('%s\n', pcmd('nft list chain inet fw4 output 2>/dev/null')); +printf('%s\n', anon_text(fw4_output_txt)); // --- recent mwan3 log --- printf('=== Recent mwan3 Log (last 200 lines) ===\n'); -printf('%s\n', pcmd('logread 2>/dev/null | grep -E "mwan3track\\[|mwan3-hotplug\\[|mwan3-init\\[|mwan3\\[" | tail -200')); +printf('%s\n', anon_text(log_txt)); printf('=== End of Report ===\n'); diff --git a/mwan3-nft/files/usr/sbin/mwan3rtmon b/mwan3-nft/files/usr/sbin/mwan3rtmon index 4059419c5..c0645a04b 100755 --- a/mwan3-nft/files/usr/sbin/mwan3rtmon +++ b/mwan3-nft/files/usr/sbin/mwan3rtmon @@ -5,7 +5,8 @@ import * as rtnl from "rtnl"; import * as uloop from "uloop"; import * as uci from "uci"; import * as ubus from "ubus"; -import { open, readfile, popen, stat } from "fs"; +import { open, readfile, popen, stat, mkdir, rename, unlink } from "fs"; +import { log_open, log_verbose, log_msg as mlog, ucibool, is_default_equivalent } from 'mwan3.common'; const RTM_NEWROUTE = rtnl.const.RTM_NEWROUTE; const RTM_DELROUTE = rtnl.const.RTM_DELROUTE; @@ -32,6 +33,7 @@ let iface_list = []; // [ { name, family, tid, enabled, has_tracking } ] let active_chains = {}; // { chain_name: true } cached from nft let connected_timer = null; // debounce timer for connected set rebuilds let last_connected_fingerprint = ""; // sorted element list from last populate call +let route_state_timer = null; // debounce timer for ROUTE_STATUS refresh let ubus_conn; // --------------------------------------------------------------------------- @@ -40,7 +42,7 @@ let ubus_conn; function log_msg(level, msg) { - warn(`mwan3rtmon[${family_name}] ${level}: ${msg}\n`); + mlog(level, `[${family_name}] ${msg}`); } // --------------------------------------------------------------------------- @@ -52,6 +54,10 @@ function load_config() { let cur = uci.cursor(); cur.load("mwan3"); + // Verbose logging gate + + log_verbose(ucibool(cur.get("mwan3", "globals", "verbose_logging") ?? "0")); + // MMX mask mmx_mask = cur.get("mwan3", "globals", "mmx_mask") ?? "0x3F00"; @@ -219,9 +225,17 @@ function nft_exec(cmd) { return true; } +// Run a list of nft commands as a single nft -f transaction. The batch +// file is per-family: one mwan3rtmon instance runs per address family +// and each instance is single-threaded, so the family suffix alone +// keeps the two instances' batch files apart without locking. The file +// is removed on every exit path after creation. nft runs through a +// read-mode popen with stderr folded into stdout so the failure log +// carries nft's own error text. + function nft_batch(cmds) { - let tmpfile = "/var/run/mwan3/mwan3rtmon_nft_batch"; + let tmpfile = "/var/run/mwan3/mwan3rtmon_nft_batch." + family_name; let f = open(tmpfile, "w", 0600); if (!f) { log_msg("warn", "cannot open nft batch file"); @@ -232,10 +246,13 @@ function nft_batch(cmds) { f.close(); let p = popen("nft -f " + tmpfile + " 2>&1"); - if (!p) + if (!p) { + unlink(tmpfile); return false; + } let out = p.read("all"); let rc = p.close(); + unlink(tmpfile); if (rc != 0) { log_msg("warn", "nft batch failed: " + (out ?? "")); return false; @@ -323,6 +340,126 @@ function is_linklocal_route(route) { match(route.dst, /^169\.254\./) != null)); } +// Does this interface's own routing table hold a usable default route? +// +// This is the datapath truth: the interface's fwmark ip rule looks up +// exactly this table, so an interface whose table N holds no default route +// black-holes the traffic mwan3 marks to it, even when the main table still +// has a default via the same device. This mirrors the predicate the rpcd +// backend uses for has_default and the one mwan3-manage-rules.uc check-route +// uses. 'routes' is a single full route dump for this family (all tables). + +function table_has_default(routes, tid, dev) { + + for (let r in routes) + if (r.table == tid && is_default_route(r) && r.oif == dev) + return true; + return false; +} + +// The interfaces this rtmon instance is responsible for: enabled, tracked, +// of this family, and currently bound to an l3 device. iface_list is built +// by load_config() and already carries name, family, tid, enabled, and +// has_tracking. The device comes from the same ubus network.interface dump +// build_dev_table_map() uses. An interface with no l3_device yet is skipped; +// it gets evaluated later when its device appears and routes change. + +function tracked_interfaces() { + + let ifaces = []; + if (!ubus_conn) + return ifaces; + let dump = ubus_conn.call("network.interface", "dump"); + if (!dump || !dump.interface) + return ifaces; + + let dev_by_name = {}; + for (let intf in dump.interface) + if (intf.l3_device) + dev_by_name[intf.interface] = intf.l3_device; + + for (let iface in iface_list) { + if (iface.family != family_name || !iface.enabled || !iface.has_tracking) + continue; + let dev = dev_by_name[iface.name]; + if (dev == null) + continue; + push(ifaces, { name: iface.name, tid: iface.tid, dev: dev }); + } + return ifaces; +} + +// Write the interface's ROUTE_STATUS file, only when the value changes, and +// log the transition. The write is atomic (temp file + rename within the +// same tmpfs directory) so the tracker never reads a torn value. The +// tracker creates these directories, but rtmon may evaluate an interface +// before its tracker has, so mkdir best-effort first; it fails harmlessly +// if the directory already exists. + +function write_route_state(ifname, state) { + + let path = MWAN3TRACK_STATUS_DIR + "/" + ifname + "/ROUTE_STATUS"; + let cur = trim(readfile(path) ?? ""); + if (cur == state) + return; + + mkdir(MWAN3TRACK_STATUS_DIR, 0700); + mkdir(MWAN3TRACK_STATUS_DIR + "/" + ifname, 0755); + + let tmp = path + ".tmp"; + let f = open(tmp, "w", 0644); + if (!f) { + log_msg("warn", "cannot open ROUTE_STATUS for " + ifname); + return; + } + f.write(state + "\n"); + f.close(); + + if (!rename(tmp, path)) { + unlink(tmp); + log_msg("warn", "cannot commit ROUTE_STATUS for " + ifname); + return; + } + + // A missing default route is abnormal, not routine failover, so log the + // loss at warn and the return to normal at notice. + + log_msg(state == "down" ? "warn" : "notice", ifname + " default route " + state); +} + +// Recompute and persist every responsible interface's route state from one +// route dump. The dump spans all tables for this family; table_has_default +// filters to each interface's own table. Called at startup, on SIGHUP, and +// debounced from the route-event handler. + +function refresh_route_states() { + + let ifaces = tracked_interfaces(); + if (length(ifaces) == 0) + return; + + let routes = rtnl.request(RTM_GETROUTE, NLM_F_DUMP, { family: family_num }) ?? []; + + for (let iface in ifaces) + write_route_state(iface.name, + table_has_default(routes, iface.tid, iface.dev) ? "up" : "down"); +} + +// Arm the debounced refresh. Bursts of route events (PD renewal, reconnects, +// the boot-time route flood) collapse into a single sweep, exactly as the +// connected-set rebuild is debounced. + +function arm_route_state_refresh() { + + if (!route_state_timer) + route_state_timer = uloop.timer(DEBOUNCE_MS, () => { + refresh_route_states(); + route_state_timer = null; + }); + else + route_state_timer.set(DEBOUNCE_MS); +} + // Get all routes from main table (and extra lookup tables) via netlink function get_all_routes() { @@ -370,8 +507,18 @@ function populate_connected_set() { let seen = {}; let elements = []; + + // Main-table routes only. get_all_routes() also returns the extra lookup + // tables for the per-interface mirroring, but their routes belong to + // mwan3_custom_v4/v6, which grants the identical bypass and is maintained + // on those tables' own route events. Sweeping them in here as well would + // strand them: extra-table events never arm this rebuild, so a route + // deleted from its table would stay in the connected set indefinitely. + for (let route in routes) { - if (is_cidr_route(route) && !is_default_route(route) && !is_linklocal_route(route) && !seen[route.dst]) { + if (route.table != RT_TABLE_MAIN) + continue; + if (is_cidr_route(route) && !is_default_equivalent(route) && !is_linklocal_route(route) && !seen[route.dst]) { push(elements, route.dst); seen[route.dst] = true; } @@ -456,7 +603,7 @@ function repopulate_custom_sets() { if (!extra_table_set[r.table]) continue; let dst = r.dst; - if (dst == null || is_default_route(r) || is_linklocal_route(r)) + if (dst == null || is_default_equivalent(r) || is_linklocal_route(r)) continue; if (!seen[dst]) { push(cmds, "add element inet mwan3 " + setname + " { " + dst + " }"); @@ -478,7 +625,7 @@ function repopulate_custom_sets() { function handle_custom_set_event(route, is_new) { let dst = route.dst; - if (dst == null || is_default_route(route) || is_linklocal_route(route)) + if (dst == null || is_default_equivalent(route) || is_linklocal_route(route)) return; let setname = (family_num == AF_INET) ? "mwan3_custom_v4" : "mwan3_custom_v6"; @@ -536,6 +683,10 @@ function handle_route_event(event) { connected_timer.set(DEBOUNCE_MS); } + // Re-evaluate every interface's default-route state on any routing change. + + arm_route_state_refresh(); + // -- Update per-interface routing tables -- build_dev_table_map(); @@ -594,6 +745,7 @@ function main() { // Parse args family_name = ARGV[0] ?? "ipv4"; + log_open("mwan3rtmon"); if (family_name == "ipv6") { if (stat("/proc/sys/net/ipv6") == null) { @@ -610,6 +762,7 @@ function main() { } load_config(); + log_msg("notice", "started"); // Connect to ubus @@ -653,6 +806,9 @@ function main() { log_msg("debug", "populating initial per-interface routes"); populate_iface_routes(); + log_msg("debug", "initial route-state sweep"); + refresh_route_states(); + // Signal handlers for clean shutdown uloop.signal("SIGTERM", () => { @@ -669,8 +825,23 @@ function main() { log_msg("debug", "received SIGHUP, reloading config"); load_config(); repopulate_custom_sets(); + refresh_route_states(); }); + // Slow periodic re-sweep as defense in depth: bounds recovery if a + // route-state change ever arrives without an accompanying main-table + // route event (a coalesced or missed netlink multicast). It is cheap + // and silent when nothing moves, since write_route_state only writes + // on an actual change. + + function schedule_periodic_route_sweep() { + uloop.timer(20000, () => { + refresh_route_states(); + schedule_periodic_route_sweep(); + }); + } + schedule_periodic_route_sweep(); + log_msg("debug", "entering event loop"); uloop.run(); diff --git a/mwan3-nft/files/usr/sbin/mwan3track b/mwan3-nft/files/usr/sbin/mwan3track index 05d7a6b48..03718fad9 100755 --- a/mwan3-nft/files/usr/sbin/mwan3track +++ b/mwan3-nft/files/usr/sbin/mwan3track @@ -299,7 +299,7 @@ main() { trap config_reload HUP load_tracking_config - local sleep_time result ping_status loss latency + local sleep_time result ping_status loss latency route_state route_state_prev probe_ips mwan3_list_track_ips() { track_ips="$track_ips $1" @@ -352,6 +352,7 @@ main() { local lost=0 local turn=0 local failed_hosts="" + route_state_prev="" firstconnect while true; do @@ -391,7 +392,26 @@ main() { continue fi sleep_time=$interval - for track_ip in $track_ips; do + + # An interface can be reachable to its tracking target yet hold no + # usable default route in its own table, in which case mwan3 marks + # traffic to it and the per-interface backstop drops it. mwan3rtmon + # writes the interface's default-route state to ROUTE_STATUS; treat + # "down" as a failed round and skip probing so the existing score + # machinery carries the interface offline, and back online when the + # route returns. An absent file means no route monitor has evaluated + # this interface, so probe exactly as before. + + route_state=$(cat "$MWAN3TRACK_STATUS_DIR/$INTERFACE/ROUTE_STATUS" 2>/dev/null) + probe_ips=$track_ips + if [ "$route_state" = "down" ]; then + probe_ips="" + [ "$route_state_prev" != "down" ] && LOG warn "Interface $INTERFACE ($DEVICE) has no usable default route in its table; treating tracking round as failed" + elif [ "$route_state_prev" = "down" ]; then + LOG notice "Interface $INTERFACE ($DEVICE) default route restored" + fi + route_state_prev=$route_state + for track_ip in $probe_ips; do if [ $host_up_count -lt $reliability ]; then case "$track_method" in ping) diff --git a/mwan3-nft/files/usr/share/rpcd/ucode/mwan3 b/mwan3-nft/files/usr/share/rpcd/ucode/mwan3 index ac08a0523..b55cbf5e9 100644 --- a/mwan3-nft/files/usr/share/rpcd/ucode/mwan3 +++ b/mwan3-nft/files/usr/share/rpcd/ucode/mwan3 @@ -75,6 +75,16 @@ function parse_elem_val(v) { return null; } +// Allowlist for nft set elements written back to the kernel: IPv4 and IPv6 +// addresses, CIDR prefixes and hyphen-written ranges only, which is exactly +// what the apply path emits. These values are interpolated into an nft add +// element command line built with popen, so anything carrying a character +// outside this class is dropped before it can break out of the element list. + +function valid_nft_element(v) { + return match(v, /^[0-9A-Fa-f:.\/-]+$/) != null; +} + function count_nftset_elements(setname) { if (!match(setname, /^[a-zA-Z0-9_-]+$/)) return 0; @@ -216,6 +226,14 @@ function get_table_routes(table_id) { } function nslookup_resolve(host, type) { + + // Reject any host outside the hostname character class before it is + // interpolated into the /bin/sh -c command line below. This is the single + // chokepoint for both resolve_host and the nftset_resolve domain path, + // whose UCI-supplied domain values are otherwise unvalidated. + + if (!match(host, /^[a-zA-Z0-9._-]+$/)) + return []; const p = popen(sprintf('/bin/busybox nslookup -type=%s %s 127.0.0.1 2>/dev/null', type, host), 'r'); if (!p) return []; const raw = p.read('all'); @@ -233,7 +251,6 @@ function nslookup_resolve(host, type) { function routing_health() { const uci = cursor(); - const mmx_mask = int(uci.get('mwan3', 'globals', 'mmx_mask')) || 0x3F00; let bitcnt = 0; for (let m = mmx_mask; m; m >>= 1) bitcnt += m & 1; @@ -250,34 +267,47 @@ function routing_health() { UNREACH_BASE = 3000; } + const SRC_BASE = UNREACH_BASE + iface_max + 1; + // Build ordered interface list with 1-based index (same ordering mwan3 uses) + const iface_order = []; const iface_index = {}; + const iface_family = {}; uci.foreach('mwan3', 'interface', intf => { push(iface_order, intf['.name']); iface_index[intf['.name']] = length(iface_order); + iface_family[intf['.name']] = intf['family'] || 'ipv4'; }); // Fetch all ip rules and index by priority for O(1) lookup + const all_rules = get_ip_rules(); const rule_by_priority = {}; + const v4_rule_by_priority = {}; for (let rule in all_rules) { - if (rule?.priority != null) + if (rule?.priority != null) { rule_by_priority[rule.priority] = rule; + if (rule.family == 2) + v4_rule_by_priority[rule.priority] = rule; + } } // Pre-build the set of priorities that correspond to current UCI interfaces + const valid_priorities = {}; for (let ifname in iface_order) { const id = iface_index[ifname]; valid_priorities[IIF_BASE + id] = true; valid_priorities[FWMARK_BASE + id] = true; valid_priorities[UNREACH_BASE + id] = true; + valid_priorities[UNREACH_BASE + iface_max + 1 + id] = true; } valid_priorities[FWMARK_BASE + mmdefault - 2] = true; valid_priorities[FWMARK_BASE + mmdefault - 1] = true; // Per-interface health report + const result = {}; for (let ifname in iface_order) { const id = iface_index[ifname]; @@ -319,17 +349,38 @@ function routing_health() { })), }, }; + + // The source band is IPv4-only, so test presence against the IPv4 + // rule index alone. A device holds one source rule per global IPv4 + // address, all sharing this priority, so a populated slot means the + // band exists for this interface, not that every address is covered. + + if (iface_family[ifname] == 'ipv4') { + result[ifname].src_rule = { + present: v4_rule_by_priority[SRC_BASE + id] != null, + priority: SRC_BASE + id, + }; + } } // Stale rules: priorities in mwan3 ranges but not matching any current UCI interface + const stale = []; for (let rule in all_rules) { const p = rule?.priority; if (p == null) continue; + + // The iif and fwmark windows are mmdefault wide (the fwmark band + // carries the global blackhole and unreachable rules above + // iface_max). The unreachable band holds only ids 1..iface_max; + // the slot above it is unused and the source band follows at + // iface_max + 1 + id. + const in_iif_range = (p > IIF_BASE && p <= IIF_BASE + mmdefault); const in_fwmark_range = (p > FWMARK_BASE && p <= FWMARK_BASE + mmdefault); - const in_unreach_range = (p > UNREACH_BASE && p <= UNREACH_BASE + mmdefault); - if ((in_iif_range || in_fwmark_range || in_unreach_range) && !valid_priorities[p]) + const in_unreach_range = (p > UNREACH_BASE && p <= UNREACH_BASE + iface_max); + const in_src_range = (p > UNREACH_BASE + iface_max + 1 && p <= UNREACH_BASE + 2 * iface_max + 1); + if ((in_iif_range || in_fwmark_range || in_unreach_range || in_src_range) && !valid_priorities[p]) push(stale, { priority: p }); } @@ -339,7 +390,7 @@ function routing_health() { interfaces: result, stale_rules: stale, mwan3_active: mwan3_active, - rule_bases: { iif: IIF_BASE, fwmark: FWMARK_BASE, unreach: UNREACH_BASE }, + rule_bases: { iif: IIF_BASE, fwmark: FWMARK_BASE, unreach: UNREACH_BASE, src: SRC_BASE }, }; } @@ -391,12 +442,14 @@ function get_policies(version) { const family_filter = (version == '6') ? 'ipv6' : 'ipv4'; // Build interface family map: iface_name -> 'ipv4'/'ipv6' + const iface_family = {}; uci.foreach('mwan3', 'interface', intf => { iface_family[intf['.name']] = intf['family'] || 'ipv4'; }); // Build member info map: member_name -> { interface, metric, weight } + const member_info = {}; uci.foreach('mwan3', 'member', m => { member_info[m['.name']] = { @@ -415,6 +468,7 @@ function get_policies(version) { if (type(use_members) != 'array') use_members = [use_members]; // Collect members matching the requested address family + const by_metric = {}; for (let mname in use_members) { const m = member_info[mname]; @@ -431,6 +485,7 @@ function get_policies(version) { if (!length(keys(by_metric))) return; // Find the lowest metric with at least one online member + const sorted_metrics = sort(keys(by_metric), (a, b) => int(a) - int(b)); let active_metric = null; for (let metric in sorted_metrics) { @@ -444,6 +499,7 @@ function get_policies(version) { } // Sum weights of online members at the active metric for percent calculation + let total_weight = 0; if (active_metric != null) { for (let m in by_metric[active_metric]) { @@ -498,10 +554,12 @@ function interfaces_status(request) { 'packetloss': get_int(ifname, sprintf('LOSS_%s', ip)), }); } + // Derive check_quality from LATENCY file content rather than UCI. // UCI may be stale if check_quality was changed without restarting mwan3track. // mwan3track only writes LATENCY_* files when check_quality=1, so file // presence with content is the authoritative indicator of actual state. + let check_quality = false; for (let tip in track_ips) { const lat = get_str_raw(ifname, sprintf('LATENCY_%s', tip.ip)); @@ -694,7 +752,7 @@ const methods = { if (type(entries) != 'array') entries = [entries]; for (let e in entries) { const ev = ltrim(rtrim(e, ' \t\r\n'), ' \t\r\n'); - if (length(ev) > 0) push(elements, ev); + if (length(ev) > 0 && valid_nft_element(ev)) push(elements, ev); } if (cfg.loadfile) { const content = readfile(cfg.loadfile); @@ -703,7 +761,7 @@ const methods = { const ci = index(line, '#'); if (ci >= 0) line = substr(line, 0, ci); line = ltrim(rtrim(line, ' \t\r\n'), ' \t\r\n'); - if (length(line) > 0) push(elements, line); + if (length(line) > 0 && valid_nft_element(line)) push(elements, line); } } } diff --git a/mwan3-nft/files/usr/share/ucode/mwan3/common.uc b/mwan3-nft/files/usr/share/ucode/mwan3/common.uc new file mode 100644 index 000000000..5e8f1e618 --- /dev/null +++ b/mwan3-nft/files/usr/share/ucode/mwan3/common.uc @@ -0,0 +1,107 @@ +'use strict'; + +// common.uc: the shared mwan3 ucode library, the ucode analogue of common.sh. +// Each shared ucode routine is defined here so it lives, and is tested, in one +// place. + +import { openlog, syslog, LOG_PID, LOG_DAEMON } from 'log'; + +// ---- Logging --------------------------------------------------------------- +// +// All mwan3 ucode logs through these functions so the sink, tag format and +// verbose gate live in one place. Output goes to syslog, the same destination +// logger(1) reaches, so it appears in logread regardless of how procd handles +// the process stderr. + +let verbose = false; + +// Map the level names mwan3 uses (the shell's warn/error vocabulary and the +// canonical syslog names) onto the exact tokens parse_priority() accepts. It +// rejects "warn" and "error", so an unmapped name would be dropped; an unknown +// level falls back to notice so a mislabelled message still logs. + +const PRIO = { + emerg: "emerg", alert: "alert", crit: "crit", + err: "err", error: "err", + warn: "warning", warning: "warning", + notice: "notice", info: "info", debug: "debug", +}; + +// Open the syslog connection with the given identity. LOG_PID appends the pid +// so the tag reads ident[pid], matching the shell's "${SCRIPTNAME}[$$]" +// convention. Facility daemon suits these long-running and service-invoked +// components. + +function log_open(ident) { + openlog(ident, LOG_PID, LOG_DAEMON); +} + +// Set the verbose gate. The caller reads mwan3.globals.verbose_logging and +// pushes the parsed bool here; debug is suppressed unless verbose, exactly as +// common.sh:LOG() gates the debug facility behind MWAN3_VERBOSE_LOGGING. + +function log_verbose(enable) { + verbose = enable; +} + +// Emit one message. debug is dropped unless verbose; every other level always +// logs. Message content is passed as a %s argument so it is never treated as a +// format string. + +function log_msg(level, msg) { + if (level == "debug" && !verbose) + return; + syslog(PRIO[level] ?? "notice", "%s", msg); +} + +// ---- Config helpers -------------------------------------------------------- + +// Parse a UCI bool the way the shell's config_get_bool does: 1/on/true/yes/ +// enabled are true, everything else is false. Kept here so every ucode +// consumer interprets a UCI bool identically to the shell. It reads no config +// itself, so it does not couple the module to uci. + +function ucibool(val) { + switch (val) { + case 'yes': + case 'on': + case 'true': + case 'enabled': + return true; + default: + return !!int(val); + } +} + +// ---- Route classification -------------------------------------------------- + +// The width classifier for the bypass sweeps: a route is default-equivalent +// when its destination is absent (a literal default) or its prefix is wide +// enough that it can only exist as a component of an in-substance default +// route - /2 or wider for IPv4, /3 or wider for IPv6, which covers the +// split-half pairs VPN clients install and the 2000::/3 global-unicast +// aggregate. Each route is judged alone, so a half-installed pair is still +// excluded. The family comes from iptoarr, the inet_pton wrapper, so no +// address text is hand-parsed; a destination it cannot parse is not +// default-equivalent. The strict literal-default predicate the mirroring and +// route-state paths use is deliberately not widened and lives on beside this. + +function is_default_equivalent(route) { + let dst = route.dst; + if (dst == null) + return true; + let slash = index(dst, "/"); + if (slash < 0) + return false; + let a = iptoarr(substr(dst, 0, slash)); + if (a == null) + return false; + return +substr(dst, slash + 1) <= (length(a) == 16 ? 3 : 2); +} + +// ---- Exports --------------------------------------------------------------- + +// The module's public interface. Declared as an export list rather than inline +// on each function, which this ucode build does not accept. + +export { log_open, log_verbose, log_msg, ucibool, is_default_equivalent }; diff --git a/mwan3-nft/mwan3-nftables-internals.html b/mwan3-nft/mwan3-nftables-internals.html index 4a579af2c..bdc54637e 100644 --- a/mwan3-nft/mwan3-nftables-internals.html +++ b/mwan3-nft/mwan3-nftables-internals.html @@ -69,7 +69,7 @@

mwan3 nftables User and Developer Reference

-

mwan3 version: 3.6.9

+

mwan3 version: 3.6.12

Covers the nftables port of the mwan3 multi-WAN policy routing framework.

mmdefault-1MM_UNREACHABLERoutes to unreachable
mmdefault (all bits set)MMX_DEFAULT (= MMX_MASK)0x3F00 = use default routing
-

Bit Spreading: mwan3_id2mask()

+

Bit Spreading: mwan3_id2mask()

Interface IDs (sequential integers 1, 2, 3...) are mapped onto the mask bits using mwan3_id2mask(). This "spreads" the ID's bits into only the positions where the mask has a 1-bit. For example, with mask 0x3F00:

Interface 1 (binary 000001) -> 0x0100   (bit 8 set)
 Interface 2 (binary 000010) -> 0x0200   (bit 9 set)
@@ -431,6 +441,8 @@ chain mwan3_or_ct_0x0100 { ct mark set ct mark & MMX_MASK_COMPLEMENT | 0x010
 
Each group appears in two distinct contexts with different match directions:

 
Destination match - mwan3_connected / mwan3_custom / mwan3_dynamic chains: These chains are jumped from mwan3_prerouting and mwan3_output before the user rules chain. Rules match ip daddr @set / ip6 daddr @set. If the packet's destination falls in one of the sets, the packet is stamped MMX_DEFAULT (mwan3 mark bits cleared to zero) and returned - the user rules chain is never reached. Traffic destined for directly-connected, custom-table, or explicitly listed bypass networks is not policy-routed.

 
Source match - mwan3_iface_in_* chains: Each per-interface chain includes source-match rules against all three set groups (ip saddr @set, scoped to the interface's address family). If an inbound WAN packet's source address is in any of the sets, it is stamped MMX_DEFAULT. Traffic arriving on a WAN interface from a connected or bypassed address does not get the WAN's interface mark, so replies to it use the main routing table rather than the WAN-specific policy route.

+
Routes wide enough to be a component of a default route are never swept into the connected or custom sets. A destination prefix of /2 or shorter for IPv4, or /3 or shorter for IPv6, is treated as default-equivalent and excluded by every producer that fills those sets: the startup seeding in mwan3_set_connected_sets() and mwan3_set_custom_sets(), and each of mwan3rtmon's three sweeps. Each route is judged on its own, so a client that has installed only half of a 0.0.0.0/1 plus 128.0.0.0/1 pair is excluded just the same. The 224.0.0.0/3 element of mwan3_connected_v4 is unaffected: it is added directly rather than swept from a route, and the IPv4 threshold stops at /2.

+
Without this exclusion a single such route would silently disable mwan3 for its whole address family. The bypass chains stamp MMX_DEFAULT before the user rules and again after them, so a covering route in the connected set would override rule steering, sticky pinning and a fail-closed last_resort policy used as a kill switch, and the source-match rules in the mwan3_iface_in_* chains would strip inbound reply-path classification from every WAN at once. No genuine network route is as wide as these thresholds. A deliberately broad bypass is still available, but it has to be asked for explicitly through globals.bypass_network, which populates the mwan3_dynamic sets and is never swept from the routing table.

 
Per-Interface Chain Detail

 
Each mwan3_iface_in_<name> chain handles packets arriving on a specific WAN device:

 
    
@@ -477,8 +489,12 @@ chain mwan3_or_ct_0x0100 { ct mark set ct mark & MMX_MASK_COMPLEMENT | 0x010
 
  1. Routes from any additional tables listed in UCI globals.rt_table_lookup
    2. 
 
 
    The MWAN3_ROUTE_LINE_EXP sed expression strips transient attributes (linkdown, offload, expiry timestamps, error codes) before copying routes. Routes whose device matches the mwan3 interface's network device keep their original dev clause; the interface's default gateway (default via ... dev ...) is the critical entry that directs all policy-routed traffic to the correct WAN path.
    
-
    5.2 Three ip Rule Tiers Per Interface
    
-
    mwan3_create_iface_rules() installs three ip rule entries for each mwan3 interface. For IPv4 interfaces these are installed via ip rule; for IPv6 via ip -6 rule. All three tiers are installed together when the interface comes up (ifup hotplug event) and removed together when it goes down (ifdown).
    
+
    Every mwan3-managed interface must carry a literal default route out of its own device in its own per-interface table: 0.0.0.0/0 for an IPv4 interface, ::/0 for an IPv6 one. This is a requirement rather than a recommendation. mwan3 is a multi-WAN manager and its interfaces are by definition whole-internet paths that policies choose between, so an interface that cannot act as a default route cannot do the job mwan3 exists to give it. The requirement is capability, not use: the interface must be able to carry the whole internet even if no rule ever steers everything to it. For a WireGuard interface this means allowed_ips '0.0.0.0/0' (and ::/0 for an IPv6 interface) together with route_allowed_ips '1', which installs a literal default route on the interface.
    
+
    An interface whose own table holds no literal default route out of its own device is taken offline, deliberately. The route monitor evaluates each tracked interface's table and writes the verdict to the interface's ROUTE_STATUS file, logging the loss at warn level as <interface> default route down. The tracker reads that file once per round and treats down as a failed round, so the ordinary score and hysteresis carry the interface offline, and back online when the route returns. An interface that reads offline with that message in the log is not misbehaving; it is this requirement being enforced, and the remedy is to give the interface a default route.
    
+
    Some VPN clients install routes that cover the whole internet without being a default route, and the shape is often the provider's choice rather than yours. OpenVPN's redirect-gateway def1 installs 0.0.0.0/1 and 128.0.0.0/1 in place of a default, split-tunnel WireGuard configurations do the same, and some IPv6 providers push the global-unicast prefix 2000::/3. None of these is a literal default route, so none of them satisfies the requirement. Where the client configuration is under your control, prefer plain redirect-gateway to redirect-gateway def1, which installs a genuine default. Where it is not, such as a server-pushed def1 pair or a server-pushed 2000::/3, add a literal default route to the interface yourself as a static route in the interface's configuration; the pushed routes are left untouched and the interface then satisfies the route-state check.
    
+
    5.2 Four ip Rule Tiers Per Interface
    
+
    mwan3_create_iface_rules() installs the Tier 1, Tier 2 and Tier 4 ip rule entries for each mwan3 interface. For IPv4 interfaces these are installed via ip rule; for IPv6 via ip -6 rule, with Tier 4 installed for IPv4 interfaces only. All three are installed when the interface comes up (ifup or connected hotplug event) and removed when it goes down (ifdown).
    
+
    Tier 3 (the per-WAN unreachable backstop) has a separate lifecycle managed by mwan3_set_member_backstops(). One Tier 3 rule is installed for every configured interface at service start and on every reload, and removed only when the service stops. The Tier 3 rule is therefore present for all configured WANs, including those that are currently offline or disabled, for as long as the service is running.
    
 
    Tier 1 - iif lookup (priority id + iif_rule_base, default id + 1000):
    
 
    ip rule add pref <id+1000> iif <device> lookup <id>
    
 
    Matches packets arriving on this interface's network device, regardless of mark. Routes them by looking up the interface's own routing table. This handles traffic that arrived on a WAN interface and needs to exit the router via the same WAN - the interface's table contains the WAN's default gateway. Without this rule, reply packets from the router (such as responses to health-check probes from the WAN's remote host) would reach the routing decision without a mark, fall through to the main table, and potentially exit via a different WAN.
    
@@ -488,6 +504,11 @@ chain mwan3_or_ct_0x0100 { ct mark set ct mark & MMX_MASK_COMPLEMENT | 0x010
 
    Tier 3 - fwmark unreachable (priority id + unreachable_rule_base, default id + 3000):
    
 
    ip rule add pref <id+3000> fwmark <mark>/<MMX_MASK> unreachable
    
 
    Matches the same fwmark as Tier 2 but returns ICMP unreachable instead of routing. Sits below Tier 2 in priority (higher priority number = lower precedence). Serves as a safety net: if the Tier 2 lookup finds an empty or incomplete routing table (e.g. the default route is transiently absent during route installation), the packet receives an explicit unreachable response instead of silently falling through to the main routing table and potentially leaking out a different WAN.
    
+
    Tier 4 - source address lookup (priority id + unreachable_rule_base + MWAN3_INTERFACE_MAX + 1, default id + 3061, IPv4 only):
    
+
    ip rule add pref <id+3061> from <address> lookup <id>
    
+
    Matches router-originated traffic by its source address rather than by mark or by arrival device. mwan3_create_iface_rules() delegates to mwan3-manage-rules.uc add-src, which dumps the interface's addresses and installs one rule per global-scope IPv4 address carried on the device, all at the same priority, so a WAN holding several addresses gets several rules.
    
+
    The tier exists because a sender that performs its own route lookup outside netfilter's view never acquires an mwan3 mark, so Tier 2 cannot steer it. Its packets follow the main table out of the default WAN even though the source address they carry belongs to a different WAN, and are dropped upstream by BCP38 filtering or masqueraded to the wrong address. WireGuard is the common case: a handshake arriving on a non-default WAN is answered from that WAN's address but leaves by the default WAN, so the handshake never completes. Any locally originated flow that no user rule classifies shares the gap. With the source rule in place the route lookup resolves back to the arrival WAN's own table and the reply keeps a source address that is valid on the path it takes.
    
+
    The tier sits below Tier 3 in priority, so marked traffic still matches Tier 2 or the Tier 3 backstop first and policy verdicts and the fail-closed backstops behave exactly as they did without it. The rules are inert for forwarded traffic, whose source is a downstream address, and for a single-WAN setup.
    
 
    5.3 Global Policy Rules
    
 
    mwan3_set_general_rules() installs two rules that are not tied to any specific interface. These handle the mark values reserved for policy last_resort actions:
    
 
    Blackhole rule (priority MM_BLACKHOLE + fwmark_rule_base):
    
@@ -497,7 +518,7 @@ chain mwan3_or_ct_0x0100 { ct mark set ct mark & MMX_MASK_COMPLEMENT | 0x010
 
    ip rule add pref <priority> fwmark <MMX_UNREACHABLE>/<MMX_MASK> unreachable
    
 
    Returns ICMP unreachable for packets marked with MMX_UNREACHABLE. This is the default last_resort for policies with no available members.
    
 
    With MMX_MASK = 0x3F00 (6 mask bits): MM_BLACKHOLE = 61, priority 61 + 2000 = 2061; MM_UNREACHABLE = 62, priority 62 + 2000 = 2062. Both fall within the fwmark tier, above all per-interface Tier 2 entries.
    
-
    5.4 MMX_DEFAULT Fall-Through
    
+
    5.4 MMX_DEFAULT Fall-Through
    
 
    Packets marked with MMX_DEFAULT (= MMX_MASK, all mask bits set) do not match any per-interface fwmark rule or the global blackhole/unreachable rules. They fall through to the kernel's standard policy database, which includes the main routing table at priority 32766. Traffic stamped MMX_DEFAULT is routed normally via the main table without any mwan3 policy steering. This is the intended path for traffic destined for directly-connected, custom-table, or dynamically bypassed networks (see the bypass set groups in Section 4).
    
 
    5.5 Configurable Rule Base Priorities
    
 
    The three rule-base offsets are configurable in UCI config globals:
    
@@ -506,13 +527,16 @@ chain mwan3_or_ct_0x0100 { ct mark set ct mark & MMX_MASK_COMPLEMENT | 0x010
 iif_rule_base10001001 - 1060 (one per interface)
 fwmark_rule_base20002001 - 2062 (per-interface + 2 global)
 unreachable_rule_base30003001 - 3060 (one per interface)
+none, derived30613062 - 3121 (Tier 4, one or more per IPv4 interface)
 
+
    The Tier 4 source band has no UCI option of its own. It is derived as unreachable_rule_base + MWAN3_INTERFACE_MAX + 1 + id, so it moves with unreachable_rule_base and always sits immediately above the unreachable band, separated from it by one unused priority (3061 with the defaults).
    
 
    Two ordering constraints must hold at startup:
    
 
    iif_rule_base + MWAN3_INTERFACE_MAX < fwmark_rule_base
 fwmark_rule_base + MWAN3_INTERFACE_MAX + 1 < unreachable_rule_base
    
 
    The + 1 in the second constraint accounts for the global blackhole and unreachable entries at the top of the fwmark tier (MM_BLACKHOLE + fwmark_rule_base and MM_UNREACHABLE + fwmark_rule_base). If either constraint is violated, mwan3_init logs a warning and reverts all three to 1000/2000/3000.
    
-
    mwan3_delete_face_rules() uses content-based matching to find and delete rules: it scans ip rule list for entries referencing lookup <id> to identify iif rules, and derives the fwmark value from the fwmark lookup rule before deleting the matching unreachable rule. This approach remains correct if the bases or MMX_MASK are changed between service restarts.
    
+
    mwan3_delete_iface_rules() removes the Tier 1, Tier 2 and Tier 4 rules by delegating to mwan3-manage-rules.uc delete-iface, which locates each rule by its priority offset and table ID and removes it via RTM_DELRULE; a Tier 4 rule is identified by carrying a source address at the source-band priority, so every address rule the interface holds goes in the one pass. This approach remains correct if the bases or MMX_MASK are changed between service restarts. The Tier 3 unreachable backstop is not deleted here; it persists until service stop.
    
 
    5.6 Example: Live ip rule Output
    
+
    Three WANs are configured on this router: eth1 (id 1) and eth2 (id 2) are online, while a third interface (id 3) is configured but currently offline.
    
 
    0:      from all lookup local
 1001:   from all iif eth1 lookup 1
 1002:   from all iif eth2 lookup 2
@@ -522,8 +546,13 @@ fwmark_rule_base + MWAN3_INTERFACE_MAX + 1 < unreachable_rule_base
    
+
    The two entries at 3062 and 3063 are the Tier 4 source rules, one per global-scope IPv4 address on each online WAN's device. Router-originated traffic that carries no mwan3 mark, such as a WireGuard reply, reaches them with the WAN address it was sourced from and is routed back out that WAN's own table instead of following the main table out of the default WAN.
    
+
    Interface id 3 shows the effect of the permanent Tier 3 backstops. It is offline, so it has no Tier 1 (iif) rule, no Tier 2 (fwmark 0x300/0x3f00 lookup 3) rule and no Tier 4 source rule; those are installed only while the interface is up, so there is no 1003, 2003 or 3064 entry. Its Tier 3 rule at priority 3003 is still present, because the backstops are installed for every configured WAN at service start and removed only at service stop. Traffic still marked 0x300 for the offline WAN therefore matches 3003 and is rejected as unreachable instead of falling through to the main table at priority 32766 and leaking out the default route. The backstop matches on the fwmark rather than on a device, which is why it can remain in place while the interface has no device bound.
    
 
    The corresponding per-interface routing table (ip route show table 1) contains the WAN gateway and a copy of connected routes:
    
 
    default via 192.0.2.1 dev eth1
 192.0.2.0/24 dev eth1 proto kernel scope link src 192.0.2.2
@@ -570,10 +599,18 @@ table inet mwan3 { ... }
    
 
    
 
    Shell scoping note: Functions like mwan3_id2mask and mwan3_count_one_bits receive variable names as arguments (e.g., mwan3_id2mask mmdefault MMX_MASK) and use arithmetic expansion $(($1)) to resolve them. This works in busybox ash (OpenWrt's default shell) because it uses dynamic scoping - local variables from the caller are visible in called functions.
    
 
    
-
    6.3 lib/mwan3/mwan3.sh
    
+
    6.3 usr/share/ucode/mwan3/common.uc
    
+
    Shared ucode library imported by the mwan3 ucode components, the ucode analogue of common.sh. Each shared ucode routine lives here so it is defined and tested in one place. Provides:
    
+
      
+
    • Logging (log_open(), log_verbose(), log_msg()): all mwan3 ucode logging goes through these so the sink, tag format, and verbose gate live in one place. Output goes to syslog via the log module, the same destination logger(1) reaches, so messages appear in logread regardless of how procd handles the process stderr. log_open(ident) opens the syslog connection with LOG_PID so the tag reads ident[pid], matching the shell's ${SCRIPTNAME}[$$] convention, under facility daemon. log_verbose(enable) sets the verbose gate; the caller reads mwan3.globals.verbose_logging and pushes the parsed bool. log_msg(level, msg) emits one message, dropping debug unless verbose exactly as common.sh:LOG() gates the debug facility, mapping mwan3's warn/error vocabulary and the syslog level names onto the tokens the log module accepts and passing the message as a %s argument so it is never treated as a format string.
      • 
+
    • ucibool(val): parses a UCI bool the way the shell's config_get_bool does (1/on/true/yes/enabled are true, everything else false), so every ucode consumer interprets a UCI bool identically to the shell. It reads no config itself, so it does not couple the module to uci.
      • 
+
    • is_default_equivalent(route): the width classifier for the bypass sweeps. Returns true when a route is a default in substance: no dst at all, or a prefix of /2 or shorter for IPv4, /3 or shorter for IPv6. The family is decided by iptoarr (inet_pton), so no address text is hand-parsed and a destination it cannot parse is not default-equivalent. Each route is judged alone, so half of a split pair is excluded on its own merits. The stricter literal-only predicate used by the route mirroring and ROUTE_STATUS paths is deliberately separate and is not widened.
      • 
+
    
+
    The module's public interface is declared as an export list (export { log_open, log_verbose, log_msg, ucibool, is_default_equivalent }) rather than inline on each function, which this ucode build does not accept.
    
+
    6.4 lib/mwan3/mwan3.sh
    
 
    The core engine. Contains all functions for managing nftables chains/sets/maps, ip rules, ip routes, policy creation, user rule classification, and status reporting. This is the largest file and the heart of the implementation. Sourced by init.d, hotplug, CLI, and rtmon scripts.
    
 
    See Section 7 for detailed function reference.
    
-
    6.4 etc/init.d/mwan3
    
+
    6.5 etc/init.d/mwan3
    
 
    procd service script. Handles:
    
 
      
 
    • start_service(): Loads mwan3-skeleton.nft via nft -f first (aborts if this fails), then runs the full init sequence: ensure framework, render ipsets, dnsmasq fragments, trackers, sets, general rules, ifup hotplug loop, nft chains, policies, user rules, conntrack flush (if flow offloading), dnsmasq HUP, rtmons.
      • 
@@ -594,6 +631,7 @@ mwan3_set_dynamic_sets()                     # populate dynamic sets
 mwan3_set_connected_sets()                   # populate connected sets
 mwan3_set_custom_sets()                      # populate custom sets
 mwan3_set_general_rules()                    # ip rule add (blackhole/unreachable)
+mwan3_set_member_backstops()                 # ip rule add (per-WAN unreachable backstops)
 config_foreach mwan3_ifup interface "init"   # trigger ifup hotplug per interface
 wait $hotplug_pids
 mwan3_set_general_nft()                      # populate hook chain rules
@@ -603,7 +641,7 @@ mwan3_flush_stale_conntrack()                # flush zero-mark conntrack entries
 mwan3_dnsmasq_hup()                          # HUP dnsmasq to re-populate nftset domains
 [if flow_offloading=1] flush conntrack       # force flow re-establishment under new policy
 start rtmon_ipv4 + rtmon_ipv6                # route monitor daemons
-

6.5 etc/hotplug.d/iface/25-mwan3

+

6.6 etc/hotplug.d/iface/25-mwan3

Handles interface state change events from netifd. Triggered on ifup, ifdown, connected, and disconnected actions.

Guard Checks

    @@ -619,20 +657,20 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemonsActions - - - - + + + +
    ActionOperations
    ifupUpdate peer track IP (if track_gateway enabled), create interface nft chain, create ip rules, set hotplug state, create routes, set general rules (if not init), rebuild policies (if online and not init). Signal tracker with USR2.
    ifdownSet offline state, delete map entries, delete ip rules, delete routes, delete interface nft chain. Signal tracker with USR1. Rebuild policies.
    connectedSet online state, create interface nft chain, rebuild policies.
    disconnectedSet offline state, rebuild policies.
    ifupUpdate peer track IP (if track_gateway enabled), create interface nft chain, create ip rules, create routes, set hotplug state. If not init: set general rules. If online and not init: rebuild policies (nft batch), flush worse-metric sticky sets, flush unreplied conntrack. Signal tracker with USR2.
    ifdownSet offline state. Rebuild policies + flush sticky sets (one nft batch). Delete ip rules, delete routes, delete interface nft chain. Signal tracker with USR1. Flush conntrack.
    connectedSet online state. Create interface nft chain, create ip rules (no-op if already present), create routes (no-op if already present). Rebuild policies (nft batch). Flush worse-metric sticky sets. Flush unreplied conntrack.
    disconnectedSet offline state. Rebuild policies + flush sticky sets (one nft batch). Flush conntrack.

    All actions call mwan3_flush_conntrack at end.

    ifup conditional policy rebuild: During init (MWAN3_STARTUP=init), the ifup action skips general rules and policy rebuild because the init sequence handles those after all interfaces are up. Route creation runs unconditionally on every ifup, including during init. During normal operation, policies are only rebuilt if the interface state is "online" (not for interfaces with initial_state=offline).

    -

    6.6 usr/sbin/mwan3 (CLI)

    +

    6.7 usr/sbin/mwan3 (CLI)

    User-facing command-line tool. Provides start/stop/restart/ifup/ifdown commands plus status reporting: interfaces, policies, connected, rules, status (all combined), and internal (detailed dump).

    The use command runs an arbitrary command bound to a specific interface using LD_PRELOAD with libwrap_mwan3_sockopt.so.

    The internal command shows nft list table inet mwan3 output instead of the old iptables dump.

    -

    6.7 usr/sbin/mwan3rtmon

    +

    6.8 usr/sbin/mwan3rtmon

    Route monitor daemon, reimplemented in ucode. Runs one instance per address family (ipv4/ipv6) as a procd service. Uses ucode-mod-rtnl for direct netlink access and ucode-mod-uloop for the event loop, eliminating all ip command fork+exec overhead from the original shell implementation.

    Key improvements over the shell version:

      @@ -647,7 +685,7 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemonsNew route: Adds CIDR networks to the connected set via nft add element, then replicates the route into active per-interface tables. Host routes (bare IPs without prefix length) are skipped as they are remote destinations. Schedules via the debouncer.
    • Deleted route: Schedules a debounced connected set rebuild, then removes the route from per-interface tables.
    -

    6.8 usr/share/rpcd/ucode/mwan3

    +

    6.9 usr/share/rpcd/ucode/mwan3

    ucode RPC service exposing ubus methods under the mwan3 object. Used by LuCI for the web interface.

    Uses nft -j (JSON output mode) for reliable parsing. All set/chain queries are scoped to table inet mwan3. Rule and route queries use ucode-mod-rtnl (RTM_GETRULE, RTM_GETROUTE) directly rather than spawning ip -j subprocesses.

      @@ -664,9 +702,9 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemonsmwan3.nftset_flush { set: "<name>" }: Flushes all elements from the named set via nft flush set inet mwan3 <name>. Returns {} on success or { error: "..." } on failure. Used by the Flush button on the IP Sets status tab.
    • mwan3.nftset_reload { set: "<name>" }: Flushes the set then re-adds all static entries from the UCI config ipset section (list entry values and the loadfile if configured). Returns {} on success or { error: "..." } on failure. Used by the Reload button on the IP Sets status tab.
    • mwan3.nftset_resolve { set: "<name>" }: Sends SIGHUP to dnsmasq to clear its cache, then queries each domain configured under the set's list domain option via the local dnsmasq instance (127.0.0.1). The DNS queries trigger dnsmasq's nftset= population mechanism as a side effect. Returns { resolved: N } where N is the count of domains that resolved successfully. Only applicable for sets with list domain entries. Used by the Resolve button on the IP Sets status tab.
      • -
    • mwan3.routing_health {}: Compares UCI configuration against live kernel state. Reads iif_rule_base, fwmark_rule_base, and unreachable_rule_base from UCI globals (with the same defaults and ordering constraint validation as mwan3_init) and derives iface_max dynamically from mmx_mask. Per interface, checks presence of the iif rule, fwmark lookup rule, and unreachable rule, and reports routing table default route presence. Reports stale ip rules across all three priority tiers. Returns a rule_bases object so the frontend can display configured priorities dynamically rather than assuming fixed offsets.
      • +
    • mwan3.routing_health {}: Compares UCI configuration against live kernel state. Reads iif_rule_base, fwmark_rule_base, and unreachable_rule_base from UCI globals (with the same defaults and ordering constraint validation as mwan3_init), derives iface_max dynamically from mmx_mask, and derives the source base as unreachable_rule_base + iface_max + 1. Per interface, checks presence of the iif rule, fwmark lookup rule and unreachable rule, adds the source rule for interfaces of family ipv4 (tested against the IPv4 rule set alone, since the source band is IPv4 only), and reports routing table default route presence. Reports stale ip rules across all four priority tiers. Returns a rule_bases object, the derived source base included, so the frontend can display configured priorities dynamically rather than assuming fixed offsets.
    -

    6.9 Makefile

    +

    6.10 Makefile

    Package build recipe. Key dependency changes:

    @@ -680,14 +718,16 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemons +
    Old DependencyNew Dependency
    +ucode-mod-ubus
    +ucode-mod-fs
    +ucode-mod-socket
    +ucode-mod-log
    +libnetfilter-conntrack

    PKG_BUILD_DEPENDS adds libnetfilter_conntrack and libmnl for the headers and libraries needed to compile mwan3ct.c. The runtime +libnetfilter-conntrack dependency provides the shared library that mwan3ct links against on the target.

    mwan3ipcheck.c is also compiled in the same build step. It links only against libc (inet_pton, strtol, standard string functions) and requires no additional build or runtime dependencies.

    -

    The ucode dependencies are required by the reimplemented mwan3rtmon route monitor daemon. ucode-mod-socket is required by mwan3-diag for address normalisation.

    +

    The ucode dependencies are required by the reimplemented mwan3rtmon route monitor daemon. ucode-mod-socket is required by mwan3-diag for address normalisation. ucode-mod-log provides the syslog binding used by the shared logging functions in common.uc (Section 6.3).

    Also installs:

    • mwan3-skeleton.nft to $(1)/lib/mwan3/
      • +
    • common.uc to $(1)/usr/share/ucode/mwan3/ (shared ucode library, see Section 6.3)
    • mwan3-migrate-ipset-v4.sh to $(1)/lib/mwan3/ (one-shot migration helper, deleted from the router after postinst runs it)
    • mwan3-remove-firewall-include UCI defaults to $(1)/etc/uci-defaults/ (removes legacy firewall.mwan3_reload UCI section from v3.x)
    • mwan3-lb-test to $(1)/usr/sbin/
      • @@ -704,7 +744,7 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemonsStops mwan3 a second time (by this point procd's auto-start has completed, so this stop removes auto-start's ip rules)
    • Starts mwan3 cleanly
-

6.10 usr/sbin/mwan3track

+

6.11 usr/sbin/mwan3track

Interface health probe daemon. One procd service instance is launched per enabled mwan3 interface that has tracking IPs configured. Runs as a shell script; largely unchanged from the iptables version except for the addition of track_gateway and check_quality support.

Probe methods

Configured via the track_method UCI option. Supported values: ping (default), arping, httping, nping-tcp/nping-udp/etc., nslookup. All probes are wrapped via LD_PRELOAD with libwrap_mwan3_sockopt.so (the WRAP helper), which intercepts setsockopt to set SO_BINDTODEVICE on the probe socket. This binds the probe to the physical interface device regardless of mwan3 routing marks, ensuring the probe exits on the correct WAN.

@@ -730,11 +770,11 @@ start rtmon_ipv4 + rtmon_ipv6 # route monitor daemonsLOSS_<ip>[check_quality=1 only] Packet loss as a percentage for this IP from the most recent probe round. GATEWAYGateway IP written by mwan3_update_peer_track_ip() when track_gateway=1. Read by mwan3_load_track_ips() and prepended to the probe list. -

check_quality

+

check_quality

When check_quality=1, probes capture latency (ms) and packet loss (%) per IP. Three-state evaluation: fail (loss >= failure_loss OR latency >= failure_latency), pass (loss <= recovery_loss AND latency <= recovery_latency), grey zone (neither - probe result neither increments nor decrements score). Default thresholds (1000ms/500ms/40%/10%) are conservative, suited for monitoring without frequent false failovers.

Signal handling

procd sends SIGUSR1 (ifdown event) and SIGUSR2 (ifup event) to trigger immediate state transitions without waiting for the next probe interval.

-

6.11 usr/sbin/mwan3-lb-test

+

6.12 usr/sbin/mwan3-lb-test

Load balancing distribution verifier.

mwan3-lb-test [-6] -c <client_ip> <policy_name> [ip1 ip2 ...]
 mwan3-lb-test cleanup
@@ -752,7 +792,7 @@ mwan3-lb-test cleanup

See §17.1 for context on why this tool was added and the numgen contamination issues it was designed to detect.

-

6.12 usr/sbin/mwan3-diag

+

6.13 usr/sbin/mwan3-diag

Network diagnostic report generator.

mwan3-diag

A ucode script that collects a comprehensive snapshot of the network state relevant to mwan3 operation and prints it to stdout. Intended to produce a complete, shareable report without manual redaction.

@@ -760,7 +800,7 @@ mwan3-lb-test cleanup

Before printing any output the script builds a map of every public routable IPv4 and IPv6 address present in the collected data and replaces each one with a stable placeholder -- PUB4_1, PUB4_2, PUB6_1 and so on -- throughout the entire report. The same address always receives the same placeholder, so cross-references between sections remain consistent. Private addresses (RFC1918, link-local fe80::, ULA fc00::/7, loopback) are left unchanged. The elements of user-defined nftables sets are replaced with { ... }.

See §17.2 for context.

-

6.13 usr/sbin/mwan3ct

+

6.14 usr/sbin/mwan3ct

Targeted conntrack entry flush helper.

mwan3 needs to flush conntrack entries at multiple places in the code. These flushes are designed to be as targeted as possible in order to leave other unrelated conntrack entries unmolested.

In some cases, the conntrack CLI tool is unable to perform the necessary targeted flush. For example, mwan3 needs to flush only UNREPLIED conntrack entries when an interface comes online to break the cycle where a stale mark pins traffic to the wrong WAN. conntrack -D silently ignores the -u status filter because the delete path in conntrack-tools omits the status filter from the kernel request, making UNREPLIED-only deletion impossible via the CLI.

@@ -777,7 +817,7 @@ mwan3-lb-test cleanup

Exit code 0 on success, including when no entries were matched. Errors are printed to stderr.

-

6.14 lib/mwan3/mwan3-get-addr.uc

+

6.15 lib/mwan3/mwan3-get-addr.uc

Address lookup helper.

Queries the kernel via RTM_GETADDR (netlink) to find an IP address in one of two modes:

    @@ -786,27 +826,30 @@ mwan3-lb-test cleanup

Prints the address stripped of any prefix-length to stdout and exits 0, or exits 1 if no matching address is found. Replaces ip -f inet[6] addr show dev <device> shell parsing.

-

6.15 lib/mwan3/mwan3-manage-rules.uc

+

6.16 lib/mwan3/mwan3-manage-rules.uc

Routing policy rule manager.

-

Manages ip rules (policy routing entries) via netlink (RTM_GETRULE, RTM_NEWRULE, RTM_DELRULE). Supports four modes:

+

Manages ip rules (policy routing entries) via netlink (RTM_GETRULE, RTM_NEWRULE, RTM_DELRULE, RTM_GETROUTE, RTM_GETADDR). Supports six modes:

  • check <family> <prio1> [prio2...]: returns a decimal bitmask where bit N is set if the Nth listed priority is absent from the live rule table. Used by mwan3.sh to decide which rules need to be created.
  • check-route <family> <table_id> <device>: exits 0 if a default route for the given output device exists in the specified table, 1 otherwise. Used before populating per-interface tables to detect whether the route is already present.
    • -
  • delete-iface <id> <iif_base> <fwmark_base> <unreachable_base> <mmx_mask>: removes the iif lookup, fwmark lookup, and unreachable rules for a single interface (identified by its sequential index). Each rule is located by priority and verified by type before deletion; rules not present are silently skipped.
    • +
  • delete-iface <id> <iif_base> <fwmark_base> <mmx_mask> <src_prio>: removes the iif lookup, fwmark lookup and source lookup rules for a single interface (identified by its sequential index). Each rule is located by priority and verified by type before deletion, a source rule by carrying a source address at src_prio, so every address rule the interface holds goes in the one pass; rules not present are silently skipped. The unreachable backstop rule is not removed here; it is managed by mwan3_set_member_backstops() and persists until service stop.
  • add-general <family> <bh_prio> <bh_mark> <ur_prio> <ur_mark> <mask>: adds the shared blackhole and unreachable rules common to all interfaces. Rules already present at those priorities are not re-added.
    • +
  • add-backstop <family> <prio> <mark> <mask>: adds one per-interface fwmark unreachable rule at the given priority, the Tier 3 backstop. A rule already present at that priority is not re-added.
    • +
  • add-src <device> <table_id> <prio>: adds one source lookup rule per global-scope IPv4 address carried on the device, all at the given priority, the Tier 4 return rules. IPv4 only.

Replaces ip rule add/del shell invocations that required fork-exec per operation and text parsing for presence checks.

-

6.16 lib/mwan3/mwan3-list-routes.uc

+

6.17 lib/mwan3/mwan3-list-routes.uc

Route table lister.

-

Dumps routes from a given routing table via RTM_GETROUTE and prints destination prefixes to stdout, one per line. Accepts a numeric table ID or the special value main. Filters out: default routes (0.0.0.0/0, ::/0), host routes (prefix-length 32/128), link-local prefixes (fe80::/10, 169.254.0.0/16), IPv4 multicast (first octet >= 224), and duplicate destinations.

-

Used by mwan3.sh to enumerate routes that need to be copied into per-interface routing tables.

+

Dumps routes from a given routing table via RTM_GETROUTE and prints destination prefixes to stdout, one per line. Accepts a numeric table ID or the special value main. Filters out: default-equivalent routes, host routes (prefix-length 32/128), link-local prefixes (fe80::/10, 169.254.0.0/16), IPv4 multicast (first octet >= 224), and duplicate destinations.

+

A route is default-equivalent when it has no destination at all, or its prefix is /2 or shorter for IPv4, /3 or shorter for IPv6. That covers the literal defaults 0.0.0.0/0 and ::/0 and the covering forms some VPN clients install in their place, such as a 0.0.0.0/1 plus 128.0.0.0/1 pair or the IPv6 global-unicast prefix 2000::/3; see the bypass set groups in Section 4. The test is is_default_equivalent() from common.uc (Section 7.2).

+

Used by mwan3.sh to populate the mwan3_connected_v4/v6 sets from the main table and the mwan3_custom_v4/v6 sets from each table listed in globals.rt_table_lookup.

-

6.17 lib/mwan3/mwan3-create-iface-route.uc

+

6.18 lib/mwan3/mwan3-create-iface-route.uc

Per-interface routing table populator.

Copies routes from the main routing table (and any additional tables listed in the rt_table_lookup UCI global) into a single per-interface routing table via RTM_NEWROUTE with NLM_F_CREATE | NLM_F_REPLACE. Uses ubus (network.interface dump) to map interface names to their kernel devices and table IDs; the _4/_6 suffix convention for dual-stack interface pairs is recognised. Routes already present in the target table (matched by dst/oif/gateway/priority) are skipped. When source_routing is set, the route's source address is included in the copy.

Replaces ip route add table <id> shell invocations that required fork-exec and text parsing of ip route show output.

-

6.18 usr/bin/mwan3ipcheck

+

6.19 usr/bin/mwan3ipcheck

IP address and CIDR validation binary. Accepts a single argument and prints one of four classification strings to stdout, exiting 0 on success or 1 on failure.

@@ -848,7 +891,18 @@ mwan3-lb-test cleanup
OutputExitMeaning
get_uptime [out_var]Returns system uptime in seconds (integer).
get_online_time out_var ifaceReturns how long the interface has been online.
-

7.2 Set Management Functions

+

7.2 common.uc Functions

+

Functions in the shared ucode library usr/share/ucode/mwan3/common.uc (see Section 6.3), imported by the mwan3 ucode components.

+ + + + + + + +
FunctionPurpose
log_open(ident)Opens the syslog connection for the calling component with LOG_PID and facility daemon, so the tag reads ident[pid], matching the shell components' ${SCRIPTNAME}[$$] convention.
log_verbose(enable)Sets the module-wide verbose gate. The caller reads mwan3.globals.verbose_logging (via ucibool) and pushes the parsed bool; debug messages are suppressed unless it is true, mirroring common.sh:LOG().
log_msg(level, msg)Emits one syslog message. debug is dropped unless verbose; every other level always logs. Maps mwan3's warn/error vocabulary and the syslog level names onto the tokens the log module accepts (an unknown level falls back to notice), and passes msg as a %s argument so it is never treated as a format string.
ucibool(val)Parses a UCI bool the way the shell's config_get_bool does: 1/on/true/yes/enabled are true, everything else false. Reads no config itself, so it does not couple the module to uci.
is_default_equivalent(route)Returns true when a route is a default in substance: no dst at all, or a prefix of /2 or shorter for IPv4, /3 or shorter for IPv6. The family comes from iptoarr (inet_pton), so no address text is hand-parsed and an unparseable destination is not default-equivalent. Used by every producer that fills the connected and custom bypass sets.
+

7.3 mwan3.sh Functions

+

7.3.1 Set Management Functions

@@ -862,13 +916,14 @@ mwan3-lb-test cleanup

Why are connected functions self-contained? mwan3_set_connected_ipv4/ipv6 each manage their own batch because they may be called from contexts that are not already inside a larger batch (init.d, hotplug). mwan3rtmon does not call these shell functions; it has its own independent populate_connected_set() that operates via direct netlink calls and writes to the same sets using its own nft_batch.

-

7.3 General Rule Setup

+

7.3.2 General Rule Setup

FunctionPurpose
mwan3_set_connected_ipv4Flushes and repopulates mwan3_connected_v4 from main routing table. Adds 224.0.0.0/3 for multicast. Self-contained: starts and commits its own nft batch.
+
FunctionPurpose
mwan3_set_general_rulesAdds ip rule entries for blackhole and unreachable marks (both IPv4 and IPv6) at priorities MWAN3_FWMARK_RULE_BASE + MM_BLACKHOLE and MWAN3_FWMARK_RULE_BASE + MM_UNREACHABLE. These are ip policy rules, not nftables rules.
mwan3_set_member_backstopsInstalls one static unreachable ip rule per configured interface in the MWAN3_UNREACHABLE_RULE_BASE + id priority band. These rules are created at service start and reload and removed only at stop, so traffic carrying a member's mark is rejected rather than leaking to the main routing table when that member's fwmark rule is absent (interface down, disabled, or briefly mid-rebuild). Reserved last-resort marks are never enumerated here so last_resort behaviour is unaffected.
mwan3_set_general_nftBuilds the per-mark OR setter chains via mwan3_build_or_chains_nft(), then populates all hook chain rules: IPv6 RA bypass (prerouting), vmap-dispatched connmark restore (if mark unset), jump to mwan3_ifaces_in, fib daddr type local return (prerouting, after ifaces_in), jumps to custom/connected/dynamic/rules chains, connmark save, and post-rules connected re-check. Idempotent: checks if rules already exist before adding. Uses a single batch for all operations.
-

Rules added by mwan3_set_general_nft()

+

##### Rules added by mwan3_set_general_nft()

For each of connected/custom/dynamic chains, adds mark-setting rules that match against the corresponding sets and apply MMX_DEFAULT.

For the prerouting and output hook chains, adds (in order):

    @@ -882,7 +937,7 @@ mwan3-lb-test cleanup
  1. Post-rules: jump to custom/connected/dynamic again for non-default marks

See §2 Connmark Operations for the rationale and the kernel-limitation context.

-

7.4 Interface Management

+

7.3.3 Interface Management

@@ -890,38 +945,40 @@ mwan3-lb-test cleanup - - - + + + +
FunctionPurpose
mwan3_update_iface_to_tablePopulates mwan3_iface_tbl: a space-separated string of name=id pairs for all configured interfaces, where id is the sequential table number. Called lazily (on first use) by mwan3_get_iface_id(). Also called explicitly during start_service() to prime the cache before interface chains are created.
mwan3_create_iface_nft iface deviceCreates (or flushes) mwan3_iface_in_<iface> chain. Adds rules matching on iifname and address family: source in connected/custom/dynamic → MMX_DEFAULT; otherwise → interface mark. Adds jump from mwan3_ifaces_in if not already present. Also installs a per-interface mwan3_postrouting SNAT rule for IPv6 interfaces when the snat6 UCI option is set. Stale mwan3_snat_<iface>-tagged rules from a prior incarnation are removed first. See §4.
mwan3_rebuild_iface_nft ifaceRebuilds a single interface's nft chain if the interface is enabled, the correct family is available, and the interface is currently up (verified via ubus). Used during reload_service to rebuild all interface chains within the atomic batch. Calls mwan3_create_iface_nft() after resolving the L3 device from netifd.
mwan3_delete_iface_nft ifaceRemoves the jump rule from mwan3_ifaces_in (by handle lookup), removes any mwan3_snat_<iface>-tagged rules from mwan3_postrouting (comment-tag match), then flushes and deletes the interface chain.
mwan3_delete_iface_map_entries ifaceIterates all mwan3_sticky_v[46]_* sets in the inet family, finds sets whose name ends with _<id> (this interface's id), and flushes them. Sets are flushed rather than deleted because rule chains may still reference the set name.
mwan3_create_iface_rules iface deviceAdds ip rule entries: pref id+MWAN3_IIF_RULE_BASE (iif lookup), pref id+MWAN3_FWMARK_RULE_BASE (fwmark lookup), pref id+MWAN3_UNREACHABLE_RULE_BASE (fwmark unreachable).
mwan3_delete_iface_rules ifaceRemoves ip rules for this interface (both IPv4 and IPv6) by delegating to mwan3-manage-rules.uc delete-iface. The ucode script locates the iif lookup rule, fwmark lookup rule, and unreachable rule by priority offset and table ID, and deletes each via RTM_DELRULE. Handles rule-base changes and mmx_mask changes transparently.
mwan3_delete_iface_map_entries ifaceIterates all mwan3_sticky_v[46]_* sets in the inet family, finds sets whose name ends with _<id> (this interface's id), and pushes a flush for each into the caller's open nft batch, so the flushes commit in the same transaction as the policy rebuild. The set enumeration reads committed kernel state and so stays outside the batch. Sets are flushed rather than deleted because rule chains may still reference the set name. Must be called within an open nft batch.
mwan3_create_iface_rules iface deviceAdds ip rule entries: pref id+MWAN3_IIF_RULE_BASE (iif lookup) and pref id+MWAN3_FWMARK_RULE_BASE (fwmark lookup). For an IPv4 interface it also delegates to mwan3-manage-rules.uc add-src for the source lookup rules at pref id+MWAN3_UNREACHABLE_RULE_BASE+MWAN3_INTERFACE_MAX+1, one per global-scope IPv4 address on the device. The per-WAN unreachable backstop at id+MWAN3_UNREACHABLE_RULE_BASE is managed separately by mwan3_set_member_backstops() and is not installed here.
mwan3_rebuild_iface_rules ifaceRebuilds a single interface's ip rules and routing table if the interface is enabled, its family is available, and it is currently up with an L3 device (verified via ubus). Called from reload_service via config_foreach, after the nft batch has committed, because ip rules and routes are kernel state that nftables does not own. Calls mwan3_create_iface_rules(), which deletes the interface's existing rules before adding them, then flushes and repopulates the routing table via mwan3_delete_iface_route() and mwan3_create_iface_route(). Unlike mwan3_rebuild_iface_nft() it does not consult the hotplug state file, so an interface that is up at netifd but tracked down keeps its routing in place.
mwan3_delete_iface_rules ifaceRemoves the iif lookup, fwmark lookup and source lookup ip rules for this interface (both IPv4 and IPv6) by delegating to mwan3-manage-rules.uc delete-iface. The ucode script locates each rule by priority offset and table ID and deletes it via RTM_DELRULE. Handles rule-base changes and mmx_mask changes transparently. The unreachable backstop at id+MWAN3_UNREACHABLE_RULE_BASE is not deleted here; it persists until service stop.
mwan3_create_iface_route ifaceCopies routes from the main routing table (and any rt_table_lookup extra tables) into the per-interface routing table by invoking mwan3-create-iface-route.uc. The ucode script reads source routes via rtnl, skips routes already present in the target table, and writes each qualifying route with RTM_NEWROUTE. Passes the source_routing UCI flag to control whether source addresses are copied.
mwan3_delete_iface_route ifaceFlushes the per-interface routing table. Unchanged.
-

Handle-Based Rule Deletion

+

##### Handle-Based Rule Deletion

nftables doesn't support deleting rules by match criteria (like iptables -D chain match...). Instead, mwan3_delete_iface_nft() uses:

handle=$($NFT -a list chain inet mwan3 mwan3_ifaces_in | \
     grep "jump mwan3_iface_in_$1" | sed -n 's/.*# handle \([0-9]*\)/\1/p')
 $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle"

The -a flag shows rule handles in comments, which can then be used for targeted deletion.

-

7.5 Policy & Load Balancing

+

7.3.4 Policy & Load Balancing

- - + +
FunctionPurpose
mwan3_set_policy member_configCallback per policy member. Tracks lowest metric and accumulates online members as iface:id:weight tuples into $policy_members. Tracks offline devices into $policy_offline_devices. Uses caller's variables (dynamic scoping).
mwan3_create_policies_nft policyCreates/flushes the mwan3_policy_<name> chain. Iterates members via mwan3_set_policy, then builds the chain: single member gets a direct mark-set rule; multiple members get a numgen rule; offline devices get out-device fallback rules; last-resort rule (unreachable/blackhole/default) is appended.
mwan3_set_policy member_configCallback per policy member. Tracks lowest metric per address family and accumulates online members as iface:id:weight tuples into $policy_members_v4 and $policy_members_v6. Uses caller's variables (dynamic scoping).
mwan3_create_policies_nft policyCreates/flushes the mwan3_policy_<name> chain. Iterates members via mwan3_set_policy, then builds the chain: single online member gets a direct mark-set rule; multiple online members get a numgen rule; last-resort rule (unreachable/blackhole/default) is appended.
mwan3_set_policies_nftBefore creating policy chains, enumerates all existing mwan3_policy_* chains in inet mwan3 and deletes any whose name is not in the current UCI policy config (orphaned chain sweep). Then iterates all policy configs and calls mwan3_create_policies_nft for each.
-

7.6 Sticky Routing

+

7.3.5 Sticky Routing

+
FunctionPurpose
mwan3_get_policy_members_for_family policy familyIterates the members of a policy config via config_list_foreach. For each member whose interface matches the requested family, resolves the interface id and computes its mark, and accumulates <id>:<mark> tuples into $_policy_member_marks. Used by the sticky path in mwan3_set_user_nft_rule() to enumerate per-member sets.
mwan3_flush_worse_metric_sticky ifaceOn recovery of a higher-priority interface, flushes the per-rule sticky sets of lower-metric same-family members of each policy the recovered interface participates in, scoped to the rules that use that policy. Set names embed the rule name and member id, so a member that appears in a separate load-balancing policy with equal-ranked peers keeps those pins untouched. Called after the policy rebuild on the connected and ifup (online) hotplug arms.
-

7.7 User Rules

+

7.3.6 User Rules

- +
FunctionPurpose
mwan3_set_user_nft_rule rule ipvBuilds and adds a single nft rule to mwan3_rules. Translates UCI config options (proto, src_ip, dest_ip, src_port, dest_port, src_iface, ipset, ipset_src) into nft match expressions. For sticky rules, calls mwan3_get_policy_members_for_family() and builds a mwan3_rule_<name> chain with per-member address sets and OR-immediate restore/save (see §8). Handles logging rules. Pre-creates missing nft sets (e.g., dnsmasq nftsets that haven't started yet) to prevent batch failures. When family is explicitly ipv4 or ipv6 but none of src_ip, dest_ip, ipset, or ipset_src are set, prepends meta nfproto ipv4/ipv6 to the nft match to prevent a bare rule from matching both address families. Also translates proto icmp to meta l4proto ipv6-icmp when family is ipv6 (in an inet table, proto 1 is ICMPv4 only). src_ip and dest_ip are validated via mwan3ipcheck; a result of "invalid" or "mixed" causes the rule to be skipped with a warning log. When a comma is present in src_ip or dest_ip, the value is wrapped in nft anonymous set syntax ({ addr1, addr2 }) for the generated match expression.
mwan3_set_user_rulesFlushes mwan3_rules chain, then iterates all rule configs for both ipv4 and ipv6, calling mwan3_set_user_nft_rule for each. Uses a single batch.
mwan3_set_user_rulesFlushes mwan3_rules chain, then iterates all rule configs in config order, calling mwan3_set_user_nft_rulefor both IPv4 and IPv6. Uses a single batch.
mwan3_set_user_iface_rules iface deviceCalled on ifup to check if the rules chain needs rebuilding (if this interface is a src_iface in any rule).
-

UCI-to-nft Match Translation

+

##### UCI-to-nft Match Translation

@@ -937,19 +994,20 @@ $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle"
UCI Optionnft Expression
proto tcpmeta l4proto tcp
use_policy balancedjump mwan3_policy_balanced
use_policy defaultmeta mark set ... | MMX_DEFAULT
-

Missing nft Set Pre-creation

+

##### Missing nft Set Pre-creation

When a user rule references an ipset (destination nft set) or ipset_src (source nft set) that doesn't exist yet in table inet mwan3, nft -f would roll back the entire batch atomically. To prevent this, mwan3_set_user_nft_rule() pre-creates any missing referenced sets with the appropriate type. The same logic applies to both ipset and ipset_src.

-

7.8 User-defined nft Set Management

+

A set is pre-created at most once per render. Kernel queries inside an open batch see pre-batch state, so the renderer carries its own record of the sets it has already added to the batch, as name=addrtype tokens reset at the start of each render. The first rule to reference a missing set settles its address type for the whole render; a later rule referencing the same name from the other family is skipped, silently when it is a family=any rule and with a warning when its family is explicit. Without the record, two rules of different families referencing one missing set each added it with a conflicting type, nft rejected the batch, and the router was left with no user rules at all.

+

7.3.7 User-defined nft Set Management

- +
FunctionPurpose
mwan3_render_config_ipsetsIterates all config ipset UCI sections and calls _mwan3_render_one_ipset for each. Creates user-defined sets in table inet mwan3. Called from start_service and reload_service.
_mwan3_nft_time_to_sec strConverts an nft human-unit timeout string (e.g. 1h, 5m, 300s) to an integer number of seconds. Recognised suffixes: d (days), h (hours), m (minutes), s (seconds, default if no suffix). Used by _mwan3_ipset_needs_delete to compare the live kernel timeout against the desired spec in a common unit.
_mwan3_ipset_needs_delete name type counter timeout maxelemReturns 0 (true) if the named set exists in table inet mwan3 AND its flags (type, counter, timeout, size) differ from the desired spec; returns 1 if the set is absent or all flags match. Called by _mwan3_render_one_ipset during reload to decide whether to queue a delete set before recreating the set with new flags. Timeout values are compared in seconds after converting nft's human-unit display (e.g. 1h, 5m) via _mwan3_nft_time_to_sec.
_mwan3_render_one_ipset nameCreates one user-defined nft set. Builds a type ipv4_addr/ipv6_addr; flags interval; auto-merge; declaration, optionally appending counter; (standalone statement, NOT a flag keyword) and timeout Ns;. On the start path, unconditionally deletes any existing set before creating. On the reload path, calls _mwan3_ipset_needs_delete and only queues a delete set when flags differ; otherwise add set is idempotent and dnsmasq-populated elements survive. Sets MWAN3_NEED_DNSMASQ_HUP if a domain-populated set is deleted. Adds inline list entry elements and loadfile contents.
_mwan3_render_one_ipset nameCreates one user-defined nft set. Builds a type ipv4_addr/ipv6_addr; flags interval; declaration; appends auto-merge; only when no timeout is configured, since auto-merge coalesces contiguous addresses into merged ranges with a single expiry, silently discarding per-element TTLs in DNS-populated sets. Optionally appends counter; (standalone statement, NOT a flag keyword) and timeout Ns;. On the start path, unconditionally deletes any existing set before creating. On the reload path, calls _mwan3_ipset_needs_delete and only queues a delete set when flags differ; otherwise add set is idempotent and dnsmasq-populated elements survive. Sets MWAN3_NEED_DNSMASQ_HUP if a domain-populated set is deleted. Adds inline list entry elements and loadfile contents.
mwan3_write_dnsmasq_fragmentsFor each config ipset with list domain entries, writes a dnsmasq confdir fragment enabling nftset=/domain/FAMILY#inet#mwan3#setname. Compares against previously written fragments; restarts dnsmasq only when content changes. Silently does nothing if no domain sets are configured.
mwan3_cleanup_orphaned_ipsetsQueries nft list table inet mwan3 for user-defined sets (those not prefixed mwan3_), compares against configured set names, and pushes delete set for any orphans. Called in reload_service after mwan3_render_config_ipsets. Prevents stale sets accumulating when a set is removed from UCI config.
-

7.9 Status Reporting

+

7.3.8 Status Reporting

@@ -960,7 +1018,7 @@ $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle"
FunctionPurpose
mwan3_report_iface_status ifaceShows interface online/offline status with uptime. Checks ip rules, nft chain existence in table inet mwan3, and default route presence.
mwan3_report_rules_v4/v6Parses nft list chain inet mwan3 mwan3_rules.
mwan3_mark_to_name markResolves a numeric mark value to an interface name (or "default"/"blackhole"/"unreachable").
-

7.10 Lifecycle & Hotplug

+

7.3.9 Lifecycle & Hotplug

@@ -968,7 +1026,7 @@ $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle" - + @@ -976,6 +1034,65 @@ $NFT delete rule inet mwan3 mwan3_ifaces_in handle "$handle"
FunctionPurpose
mwan3_ifup iface callerResolves interface status via ubus, then triggers the 25-mwan3 hotplug script with ACTION=ifup. When called from init, runs in background.
mwan3_interface_shutdown ifaceCalls hotplug shutdown then cleans track state files.
mwan3_set_iface_hotplug_state iface stateWrites state (online/offline) to status file.
mwan3_get_iface_hotplug_state ifaceReads state from status file (defaults to offline).
mwan3_flush_conntrack iface actionTwo-path conntrack flush. First, iterates the flush_conntrack UCI list for this interface: for each configured action that matches the current hotplug action, writes f to $CONNTRACK_FILE to flush the entire global conntrack table. Second, on ifdown, uses conntrack -D --mark MARK/MMX_MASK to selectively delete only the conntrack entries for this interface's fwmark. Both paths run; the global flush path runs first.
mwan3_flush_conntrack iface actionTwo-path conntrack flush. First, iterates the flush_conntrack UCI list for this interface: for each configured action that matches the current hotplug action, writes f to $CONNTRACK_FILE to flush the entire global conntrack table. Second, on ifdown and disconnected, uses mwan3ct flush --mark MARK/MMX_MASK to selectively delete only the conntrack entries for this interface's fwmark. Both paths run; the global flush path runs first.
mwan3_flush_marked_conntrackFlushes all conntrack entries that have any MMX_MASK bit set, iterating the full mwan3 id-space (IDs 1..MWAN3_INTERFACE_MAX plus default/blackhole/unreachable marks). Called from reload_service so that live flows re-enter the classification chains and are re-evaluated against new rules rather than staying pinned to a previously saved ct mark.
mwan3_update_peer_track_ip ifaceIf track_gateway is enabled, queries ifstatus for the point-to-point peer address and writes it to $MWAN3TRACK_STATUS_DIR/<iface>/GATEWAY.
mwan3_track_clean ifaceRemoves track status directory for the interface.
mwan3_flush_stale_conntrackFlushes conntrack entries with no mwan3 mark (0x0/MMX_MASK) after mwan3 restart. New connections arriving during the brief startup window before iface_in chains exist get ct mark=0; this clears them so they re-establish correctly. Called from start_service.
mwan3_flush_unreplied_conntrackFlushes all mwan3-marked conntrack entries that have never received a reply (IPS_SEEN_REPLY not set). Called after mwan3_set_policies_nft on the connected action and on ifup when status=online. Breaks the cycle where a stale mark from a previous policy state pins traffic to the wrong WAN: because no reply arrives the application keeps retrying, refreshing the conntrack timeout indefinitely. Flushing only UNREPLIED entries leaves established connections untouched.
+

7.4 mwan3rtmon Functions

+

usr/sbin/mwan3rtmon (see Section 6.8) is the ucode route-monitor daemon; one instance runs per address family (ipv4 and ipv6). It mirrors kernel routing changes into mwan3's connected and custom nft sets and into the per-interface routing tables, and maintains each tracked interface's ROUTE_STATUS file. Shared logging (log_open, log_verbose, log_msg), ucibool and is_default_equivalent are imported from common.uc (Section 7.2).

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FunctionPurpose
main()Entry point. Parses the address family from ARGV[0] (ipv4/ipv6), opens syslog with log_open, loads config, connects to ubus, and binds the rtnl route listener before running the initial dumps so route changes during the dump phase are queued rather than lost to a subscribe-vs-dump race. Runs the initial custom-set, connected-set, per-interface-route, and route-state population, installs SIGTERM/SIGINT/SIGHUP handlers (SIGHUP reloads config and re-sweeps), arms a 20-second defensive route-state re-sweep (the nested schedule_periodic_route_sweep()), then enters the uloop event loop. One process runs per family.
log_msg(level, msg)Local logging wrapper that prefixes the [family] tag and delegates to common.uc's log_msg (imported as mlog).
load_config()Loads the mwan3 UCI config: sets the verbose gate via log_verbose(ucibool(...)), reads mmx_mask and source_routing, builds extra_table_set from rt_table_lookup, and builds iface_list (name, family, sequential table id, enabled, has_tracking) in UCI declaration order. Called at startup and on SIGHUP.
build_dev_table_map()From the ubus network.interface dump, maps each enabled same-family interface's l3_device to its table id in dev_table_map.
refresh_active_chains()Caches the set of existing inet mwan3 chains from nft -j list chains inet into active_chains.
is_iface_nft_active(ifname)Returns whether mwan3_iface_in_<ifname> is present in the cached chain set.
get_active_tids()Returns a {tid: name} map of same-family interfaces whose per-interface nft chain is currently active.
get_track_status(ifname)Reads the tracker's PID and STARTED status files and returns disabled, down, paused, or active, mirroring the shell tracker-status logic.
nft_exec(cmd)Runs a single nft command, logging a warning with the command and its output on non-zero exit.
nft_batch(cmds)Writes a list of commands to a per-family batch file, /var/run/mwan3/mwan3rtmon_nft_batch.<family>, and applies them atomically with nft -f, logging on failure. One instance runs per address family and each is single-threaded, so the family suffix alone keeps the two instances' files apart without locking; the file is removed on every exit path after creation.
build_route_for_table(route, tid)Builds a route object targeting table tid, copying the relevant ROUTE_FIELDS, dropping tos == 0, and including src only when source_routing is enabled.
apply_route_to_table(route, tid, is_new)Adds (RTM_NEWROUTE, create+replace) or deletes (RTM_DELROUTE) the route in table tid via rtnl, logging add failures.
is_cidr_route(route)Returns true when the route's dst prefix length is shorter than the family maximum (32/128), i.e. a network route rather than a host route.
is_default_route(route)Returns true for a null dst, 0.0.0.0/0, or ::/0. Route mirroring and the ROUTE_STATUS check rely on this narrow meaning, so it is deliberately left strict. The bypass sweeps use the wider is_default_equivalent from common.uc instead.
is_linklocal_route(route)Returns true for a dst under fe80::/ or 169.254..
table_has_default(routes, tid, dev)Returns true if the route dump holds a default route in table tid via device dev. This is the datapath predicate behind ROUTE_STATUS: the interface's fwmark ip rule looks up exactly this table, so a table with no default black-holes the traffic marked to it even when the main table still has a default via the same device.
tracked_interfaces()Returns [{name, tid, dev}] for the enabled, tracked, same-family interfaces currently bound to an l3 device (device from the ubus dump). Interfaces with no device yet are skipped and re-evaluated on a later route change.
write_route_state(ifname, state)Atomically writes the interface's ROUTE_STATUS file (temp file + rename in the same tmpfs directory) only when the value changes, and logs the transition (down at warn, the return at notice).
refresh_route_states()Recomputes every responsible interface's route state from one full route dump and persists it via write_route_state. Called at startup, on SIGHUP, from the periodic sweep, and debounced from the route-event handler.
arm_route_state_refresh()Arms or reschedules the debounce timer that collapses a burst of route events into a single refresh_route_states sweep.
get_all_routes()Dumps routes for this family and filters to the main table plus any rt_table_lookup extra tables.
route_still_exists(route)Re-dumps and checks whether a route still exists in the main table; used to verify delete events before acting on them.
populate_connected_set()Flushes and repopulates mwan3_connected_v4/v6 from CIDR, non-default-equivalent, non-link-local routes (plus 224.0.0.0/3 for IPv4), gated by a sorted-element fingerprint so an unchanged set skips the nft write.
populate_iface_routes()Copies main and extra-table routes into the active per-interface tables: a device-matched route goes to that interface's table, other network routes broadcast to all active tables.
repopulate_custom_sets()Rebuilds mwan3_custom_v4/v6 from the rt_table_lookup tables, excluding default-equivalent and link-local routes; used at startup and on SIGHUP reload.
handle_custom_set_event(route, is_new)Mirrors a single rt_table_lookup route add or delete into mwan3_custom_v4/v6 so those destinations receive MMX_DEFAULT and bypass WAN policy selection; default-equivalent and link-local routes are skipped.
handle_route_event(event)The rtnl listener callback for RTM_NEWROUTE/RTM_DELROUTE: mirrors rt_table_lookup routes into the custom set, debounces the connected-set rebuild, arms the route-state refresh, and updates the per-interface tables, verifying delete events and gating broadcast routes on tracker status.
+

7.5 mwan3track Functions

+

usr/sbin/mwan3track (see Section 6.11) is the per-interface tracker; one shell process runs per tracked interface. It probes the interface's track_ip targets on an interval, maintains a hysteresis score, writes per-target and per-interface status files under $MWAN3TRACK_STATUS_DIR/<iface>/, and fires connected/disconnected (and the transitional connecting/disconnecting) hotplug events as the score crosses its thresholds. Signals drive its lifecycle: USR1 is ifdown, USR2 is ifup, HUP is config reload, TERM is stop.

+ + + + + + + + + + + + + + + + + + + + + + + +
FunctionPurpose
stop_subprocsKills the tracker's background jobs (the running probe and sleep subshells). Used by the signal handlers to break out of the current round.
WRAP cmd...Runs the probe command cmd under LD_PRELOAD of the mwan3 sockopt wrapper, exporting FAMILY, DEVICE, SRCIP, and FWMARK so the probe leaves via the interface's own address and is marked for its own table.
clean_upTERM handler: logs the stop, kills subprocesses, and exits.
if_downUSR1 handler: sets IFDOWN_EVENT and stops subprocesses so the loop registers the ifdown.
if_upUSR2 handler: sets IFUP_EVENT, marks the tracker started, and stops subprocesses.
config_reloadHUP handler: sets RELOAD_EVENT and stops subprocesses so the loop reloads config.
ping_test_hostEchoes the family's loopback address (127.0.0.1 or ::1), used to test whether a ping variant works.
get_ping_commandSelects the usable ping binary/variant (ping -4/-6, ping6, or /bin/ping) and echoes it; returns non-zero if none is available.
validate_track_method methodVerifies the tool for the configured track_method (ping/arping/httping/nping-*/nslookup) is installed, logging a warning and returning non-zero if not.
validate_wrapEnsures the libwrap_mwan3_sockopt shared library exists; logs an error and exits if it is missing.
disconnected [suppress]Marks the interface offline: writes STATUS/OFFLINE/ONLINE, zeroes the score, and (unless suppress is 1, as on first connect) fires the disconnected hotplug event when the previous status was online or disconnecting.
connected [firstconnect]Marks the interface online: writes STATUS/ONLINE/OFFLINE, resets the per-round counters, and fires the connected hotplug event (passing FIRSTCONNECT).
disconnectingEnters the transitional disconnecting state once and fires the disconnecting hotplug event, an early warning before the interface is declared offline.
connectingEnters the transitional connecting state once and fires the connecting hotplug event, an early signal before the interface is declared online.
disabledMarks the interface disabled and calls stopped.
firstconnectResolves the true interface, L3 device, and source IP, determines the initial status (initial_state, default online), marks the tracker started, then calls connected or disconnected accordingly. Called at startup and after ifup.
stoppedSets STARTED=0 and writes the STARTED status file; the tracker then idles until a signal.
startedSets STARTED=1 and writes the STARTED status file.
update_status track_ip state [latency loss]Writes the per-target TRACK_<ip> status file (and, when supplied, LATENCY_<ip>/LOSS_<ip>) for one probe target.
load_tracking_configReads all per-interface tracking UCI options (family, track_method, reliability, count, timeout, interval, up/down thresholds, size, max_ttl, failure/recovery intervals, check_quality and its latency/loss thresholds), clamping check_quality off for track methods that produce no latency/loss samples. Called at startup and on HUP reload.
main ifaceEntry point and probe loop. Takes the per-interface lock and writes PID, installs the signal traps, loads config and the probe-target list, then loops: each round it reads ROUTE_STATUS (a down written by mwan3rtmon counts as a failed round with no probing), probes each target with WRAP, adjusts the hysteresis score, and calls the state transitions; between rounds it services pending ifdown/ifup/reload events.
+

The nested helpers mwan3_load_track_ips and mwan3_list_track_ips, defined inside main(), assemble the probe-target list (static track_ip entries plus the discovered gateway when track_gateway is set) and prune stale per-target status files; they are not listed separately.

8. Load Balancing with numgen

The iptables version used -m statistic --mode random --probability P to distribute traffic. This required inserting rules in specific order and computing running probabilities. The nftables version uses numgen inc mod N map { ... }, which is simpler and more deterministic.

@@ -990,7 +1107,7 @@ nft add rule inet mwan3 mwan3_policy_balanced \

Kernel limitation on compound set expressions: An early implementation tried meta mark set meta mark & COMP | numgen inc mod ... to preserve non-mwan3 bits while applying the numgen result. This fails with "Operation not supported" because the kernel cannot mix two register sources (meta mark and numgen) in one set expression. The solution is to use meta mark set numgen ... directly; the meta mark & MMX_MASK == 0 guard condition ensures the mwan3 bits are already zero before the numgen result is applied.

-

Build Algorithm (in mwan3_create_policies_nft)

+

Build Algorithm (in mwan3_create_policies_nft)

  1. Iterate policy members via config_list_foreach
  2. Track lowest metric per family (v4/v6 separately); only members at the lowest metric are included
    3. @@ -998,7 +1115,6 @@ nft add rule inet mwan3 mwan3_policy_balanced \
  3. Calculate total weight = sum of all member weights
  4. If single member: direct meta mark set (no numgen overhead)
  5. If multiple members: build numgen map entries with ranges proportional to weight
    6. -
  6. Append offline device fallback rules (only if no online members)
  7. Append last-resort rule (unreachable/blackhole/default)
@@ -1029,6 +1145,8 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout; meta mark & 0x3f00 == 0x100 update @mwan3_sticky_v4_https_1 { ip saddr timeout 600s } meta mark & 0x3f00 == 0x200 update @mwan3_sticky_v4_https_2 { ip saddr timeout 600s } } +

The chain is built once, complete for every family whose traffic can reach it. A rule with an explicit family, or with a family-anchoring match element (src_ip, dest_ip, ipset, ipset_src), reaches the chain in one address family only and gets that family's arms. A family=any rule with no such element renders as a single family-agnostic line in mwan3_rules, so traffic of both families reaches its chain; the IPv4 pass therefore builds both arms, ip saddr lookups and updates against the mwan3_sticky_v4_* sets and ip6 saddr against the mwan3_sticky_v6_* sets. proto icmp is the exception that renders a line on both passes, because ICMP and ICMPv6 are distinct L4 protocols; the IPv6 pass then points its own line at the chain the IPv4 pass has already built rather than appending to it.

+

The layout is load-bearing: every lookup precedes the single fall-through to the policy, and the update rules follow it. A pinned source must OR its member mark in before the fall-through can hand the packet to the policy. A lookup placed after the fall-through would let the policy re-mark the flow first and the late lookup would then OR the old pin into the new mark, producing a value that matches no interface.

Flow for a Sticky Rule

  1. Packet arrives at mwan3_rules chain
    2. @@ -1037,6 +1155,9 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout;
  2. New source: no set match, mark stays 0. Falls through to the policy chain which assigns a mark. The matching update rule then adds saddr to the corresponding member's set with the configured timeout.
  3. After timeout seconds of inactivity the set entry expires and the source is re-evaluated at next connection.
+

Set Lifecycle

+

A sticky entry normally persists until its per-entry timeout, but two failover events also flush sticky sets directly, so a pinned source can be re-evaluated before its timeout expires. When a member fails (a soft disconnected failure with the link still up) or goes down (ifdown), that member's own sticky sets are flushed by mwan3_delete_iface_map_entries in the same nft transaction that drops it from the policy, so a source pinned to the failed member re-picks the survivor on its next new connection rather than staying nailed to a member that is gone. When a higher-priority member recovers, mwan3_flush_worse_metric_sticky flushes the per-rule sticky sets of the lower-metric same-family members of each policy the recovered member participates in, scoped to the rules that use that policy (the snap-back), so new flows return to the recovered member; an equal-ranked load-balancing policy clears nothing, because no member outranks another.

+

In every case only the sticky sets are touched, and only new flows re-pick as a result. Established connections are not moved, because they are pinned by their saved conntrack mark rather than by the sticky sets (see Sections 10.3 and 10.4).

10. Service Lifecycle and Conntrack Management

10.1 Start

@@ -1045,11 +1166,12 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout; +--> mwan3_init() compute masks +--> mwan3_ensure_nft_framework() recreate 6 internal sets, ensure chains +--> mwan3_render_config_ipsets() create user-defined sets from config ipset - +--> mwan3_write_dnsmasq_fragments() write nftset confdir fragments; restart dnsmasq if changed + +--> mwan3_write_dnsmasq_fragments() write nftset fragments; restart if changed +--> start_tracker per interface launch mwan3track +--> mwan3_update_iface_to_table() build iface->table mapping +--> mwan3_set_dynamic/connected/custom sets +--> mwan3_set_general_rules() ip rule add blackhole/unreachable + +--> mwan3_set_member_backstops() per-WAN unreachable backstop ip rules +--> mwan3_ifup per interface trigger hotplug (creates iface chains + ip rules) +--> wait for hotplug completion +--> mwan3_set_general_nft() populate hook chain rules @@ -1061,7 +1183,11 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout; +--> start mwan3rtmon (ipv4 + ipv6) route monitor daemons

10.2 Reload

/etc/init.d/mwan3 reload
-  +--> mwan3_nft_reload_start()        open batch + write preamble (flush/delete dynamic state)
+  +--> [table inet mwan3 absent]       stop + start, then return
+                                       (the skeleton load that creates the
+                                       table is skipped when every interface
+                                       is disabled)
+  +--> mwan3_nft_reload_start()        open batch, flush/delete dynamic state
   +--> mwan3_ensure_nft_framework()    recreate 6 internal sets
   +--> mwan3_render_config_ipsets()    add user-defined sets (skips delete inside batch)
   +--> mwan3_cleanup_orphaned_ipsets() delete user sets not in current config
@@ -1073,32 +1199,39 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout;
   +--> mwan3_nft_reload_commit()       commit all as single atomic kernel transaction
   +--> mwan3_update_iface_to_table()   (outside nft)
   +--> mwan3_set_general_rules()       (outside nft)
+  +--> mwan3_set_member_backstops()    (outside nft)
   +--> mwan3_write_dnsmasq_fragments() restart dnsmasq only if fragment content changed
-  +--> [if MWAN3_NEED_DNSMASQ_HUP=1]   mwan3_dnsmasq_hup() repopulate domain sets lost on flag change
+  +--> [if MWAN3_NEED_DNSMASQ_HUP=1]   mwan3_dnsmasq_hup() repopulates
+                                       domain sets lost on a flag change
   +--> [tracker count check]           if mismatch: stop + start (procd cannot add/remove
                                        service instances in reload_service)

User-defined sets (config ipset) and sticky sets (mwan3_sticky_*) survive the reload intact - they are never in the preamble delete path.

-

10.3 Interface Up (hotplug)

+

10.3 Interface Up and Tracker Recovery (hotplug)

netifd signals ifup for $INTERFACE
   +--> 25-mwan3 hotplug script
-       +--> mwan3_update_peer_track_ip()                write gateway IP (if track_gateway)
-       +--> mwan3_create_iface_nft()                    create/flush chain, add rules
-       +--> mwan3_create_iface_rules()                  ip rule add (iif, fwmark, unreachable)
-       +--> mwan3_create_iface_route()                  copy routes to per-iface table
+       +--> mwan3_update_peer_track_ip()   write gateway IP (track_gateway)
+       +--> mwan3_create_iface_nft()       create/flush chain, add rules
+       +--> mwan3_create_iface_rules()     ip rule add (iif, fwmark, source)
+       +--> mwan3_create_iface_route()     copy routes to per-iface table
        +--> mwan3_set_iface_hotplug_state "online/offline"
        +--> [if not init startup:]
-       |    +--> mwan3_set_general_rules()              ensure ip rules exist
-       |    +--> [if online:] mwan3_set_policies_nft()  rebuild policy chains
-       |                      mwan3_flush_unreplied_conntrack()  flush UNREPLIED entries
+       |    +--> mwan3_set_general_rules()          ensure ip rules exist
+       |    +--> [if online:]
+       |         +--> [nft batch:] mwan3_set_policies_nft()
+       |         +--> mwan3_flush_worse_metric_sticky()
+       |         +--> mwan3_flush_unreplied_conntrack()
        +--> procd_send_signal track_$INTERFACE USR2
-
netifd signals connected for $INTERFACE
+
connected fires for $INTERFACE (link-layer negotiation, or tracker recovery)
   +--> 25-mwan3 hotplug script
        +--> mwan3_set_iface_hotplug_state "online"
-       +--> mwan3_create_iface_nft()                    flush chain, update rules
-       +--> mwan3_set_policies_nft()                    rebuild policy chains
-       +--> mwan3_flush_unreplied_conntrack()            flush UNREPLIED entries

-
The connected event fires when an interface completes its link-layer negotiation (e.g., PPPoE authentication) after the ifup routing state is already in place. It skips rule and route creation (those are done on ifup) and goes straight to policy rebuild and the unreplied flush.

-
Automatic Gateway Tracking (track_gateway)

+       +--> mwan3_create_iface_nft()       flush chain, update rules
+       +--> mwan3_create_iface_rules()     recreate ip rules (idempotent)
+       +--> mwan3_create_iface_route()     recreate routes (idempotent)
+       +--> [nft batch:] mwan3_set_policies_nft()
+       +--> mwan3_flush_worse_metric_sticky()
+       +--> mwan3_flush_unreplied_conntrack()
+

The connected action fires on two distinct events: link-layer negotiation completion (PPPoE authentication, for example) after ifup routing state is already in place, and tracker recovery, when mwan3track judges a previously-failed interface to be working again without a physical reconnect. On a tracker recovery the interface's routing rules and route may have been removed by a prior ifdown; the connected arm recreates them unconditionally so the interface is fully routable before the policy re-selects it. On an ordinary link-negotiation event the routing is already in place and the recreate calls are harmless no-ops. The snap-back (mwan3_flush_worse_metric_sticky) returns new flows from lower-ranked same-family backups to the recovered interface; established flows on those backups are left to complete undisturbed.

+

Automatic Gateway Tracking (track_gateway)

When option track_gateway '1' is set on a config interface section, mwan3 automatically discovers the point-to-point peer/gateway IP and adds it to the tracking list at runtime. This is useful for PPPoE and other point-to-point links where the next-hop gateway IP changes on each connection and is not known in advance. Without this option, users must manually configure static track_ip addresses (typically public DNS servers) that may not test the actual link peer.

mwan3_update_peer_track_ip() queries ifstatus for the interface's ptpaddress field (the point-to-point peer IP). If found, the gateway IP is written to $MWAN3TRACK_STATUS_DIR/<iface>/GATEWAY. mwan3track reads this file and prepends the gateway IP to the front of the probe list, ensuring it is always probed first on every round regardless of the reliability threshold; static track_ip entries follow after the gateway. On interface bounce, the hotplug ifup action calls mwan3_update_peer_track_ip() again, overwriting the state file with the new peer IP.

The gateway IP is stored as ephemeral state rather than committed to UCI, preventing stale IP accumulation across reboots or gateway changes. An interface definition may specify only track_gateway and omit static tracking IPs entirely. The option is silently ignored if no next-hop peer address is found (e.g., on Ethernet WAN interfaces).

@@ -1112,31 +1245,35 @@ nft add set inet mwan3 mwan3_sticky_v4_https_2 { type ipv4_addr; flags timeout; # track_ip entries are optional when track_gateway is used # list track_ip '8.8.8.8'

The corresponding LuCI control is the "Track gateway" checkbox in the Interface tab (see Section 15.1.2).

-

10.4 Interface Down (hotplug)

+

10.4 Interface Down and Soft Failure (hotplug)

netifd signals ifdown for $INTERFACE
   +--> 25-mwan3 hotplug script
        +--> mwan3_set_iface_hotplug_state "offline"
-       +--> mwan3_delete_iface_map_entries()           flush sticky sets for this iface
-       +--> mwan3_delete_iface_rules()                 ip rule del
-       +--> mwan3_delete_iface_route()                 ip route flush table
-       +--> mwan3_delete_iface_nft()                   remove chain + jump rule
+       +--> [nft batch:] mwan3_set_policies_nft()
+       |                 mwan3_delete_iface_map_entries()
+       +--> mwan3_delete_iface_rules()         ip rule del
+       +--> mwan3_delete_iface_route()         ip route flush table
+       +--> mwan3_delete_iface_nft()           remove chain + jump rule
        +--> procd_send_signal track_$INTERFACE USR1
-       +--> mwan3_set_policies_nft()                   rebuild policies (failover)
-       +--> mwan3_flush_conntrack()                    flush conntrack entries for this iface's fwmark
+ +--> mwan3_flush_conntrack() flush this iface's ct entries +
disconnected fires for $INTERFACE (tracker-detected soft failure, link up)
+  +--> 25-mwan3 hotplug script
+       +--> mwan3_set_iface_hotplug_state "offline"
+       +--> [nft batch:] mwan3_set_policies_nft()
+       |                 mwan3_delete_iface_map_entries()
+       +--> mwan3_flush_conntrack()            flush this iface's ct entries
+

On ifdown, the policy is rebuilt before the interface's routing rules are removed, ensuring the policy drops the interface from the choosable set before its routes are gone; a packet is never tagged for an interface whose routing has already been torn down. On disconnected, the interface remains physically up so routing is left intact; only the policy and stickiness records are updated. In both cases the policy rebuild and sticky flush commit as one nft transaction, in that fixed order: the rebuild runs first so the newly built policy cannot re-seed the sticky records that the flush is about to clear. The per-interface conntrack flush follows outside the transaction as a separate operation on kernel state that nftables does not own, and immediately moves established flows off the failed interface rather than leaving them to hang until TCP retransmit timeouts expire.

10.5 Stop

/etc/init.d/mwan3 stop
   +--> service_running || exit 0
   +--> mwan3_interface_shutdown per interface   trigger ifdown hotplug
   +--> flush ip routing tables (1..MWAN3_INTERFACE_MAX)
   +--> delete ip rules in configured ranges
-  +--> flush ALL mwan3_* chains (rules removed, skeleton chains kept)
-  +--> delete dynamic chains (iface_in_*, policy_*, etc.)
-  +--> final safety flush of skeleton chains
-  +--> flush ALL mwan3_* sets; flush and delete sticky maps
   +--> $NFT delete table inet mwan3   (complete removal)
   +--> rm -rf status dirs
 
 Result: table inet mwan3 no longer exists. Clean slate.
+

The single table delete is the whole nftables teardown: one kernel transaction drops every chain, set and map mwan3 created. mwan3 previously dismantled those objects one at a time, flushing all its chains, deleting the dynamic ones, flushing its sets and deleting its sticky maps before removing what remained. That sequence dates from mwan3 operating inside table inet fw4, a table it did not own and could not drop; with its own table it is redundant work, and removing it is what makes stop fast.

10.6 Conntrack Management

mwan3 uses the Linux connection tracking table (conntrack) to persist routing decisions across the packets of a single connection. On the first packet of a new flow, mwan3 stamps the packet with an interface fwmark and saves that mark into the conntrack entry's ct mark field. All subsequent packets in the same connection restore the ct mark back to the packet mark at the start of the prerouting/output chain, bypassing policy re-evaluation entirely. This means that routing policy changes do not take effect for established connections until their conntrack entries are removed.

What mwan3 does not do is often as important as what it does:

@@ -1157,10 +1294,10 @@ Result: table inet mwan3 no longer exists. Clean slate. 
echo f > /proc/net/nf_conntrack

When flow offloading is active, the kernel fast-paths established flows directly through the flowtable, bypassing nftables entirely. Any flow that was offloaded before mwan3 started will never see the mwan3 prerouting chain and will never get a mark. Flushing everything forces all connections to re-enter the nftables pipeline and be classified correctly.

This does not apply to hardware flow offloading, which uses different kernel mechanisms and is not affected by writing to /proc/net/nf_conntrack.

-

3. Interface down - selective flush (mwan3_flush_conntrack)

-

Immediately after mwan3_set_policies_nft rebuilds the policies for a failing interface, the hotplug script calls mwan3_flush_conntrack. This function has two independent paths that both run on ifdown:

+

3. Interface down / soft failure - selective flush (mwan3_flush_conntrack)

+

After the policy is rebuilt for a failing interface, the hotplug script calls mwan3_flush_conntrack. This function has two independent paths that both run on ifdown and disconnected:

First, the UCI flush_conntrack path: if the per-interface flush_conntrack UCI option contains ifdown, mwan3 writes f to /proc/net/nf_conntrack, flushing the entire global conntrack table. This is the legacy behaviour, retained for users who need it. Because it disrupts all connections on all interfaces simultaneously, it is not set by default.

-

Second, the selective flush path, which always runs on ifdown regardless of the UCI option:

+

Second, the selective flush path, which always runs on both ifdown and disconnected regardless of the UCI option:

mwan3ct flush --mark <iface_mark>/$MMX_MASK

This removes only the conntrack entries that belong to the failing interface (those whose masked mark bits equal that interface's assigned fwmark). The effect is that TCP and UDP flows that were using the failed WAN immediately re-establish under the new failover policy rather than waiting out a TCP retransmit timeout (typically 15-30 seconds). Connections on healthy interfaces are completely undisturbed.

4. Interface up - UCI flush only

@@ -1175,6 +1312,7 @@ Result: table inet mwan3 no longer exists. Clean slate. start--if flow_offloadingyes (always) reload---- ifdownyes (always)-if flush_conntrack includes ifdown- +disconnectedyes (always)-if flush_conntrack includes disconnected- ifup-if status=onlineif flush_conntrack includes ifup- connected-yes (always)-- stop---- @@ -1191,7 +1329,7 @@ Result: table inet mwan3 no longer exists. Clean slate.
  • Load-balanced configurations: when a failed WAN recovers, the automatic ifdown flush has already cleared entries for the failed interface, but all surviving flows remain pinned to the WANs that carried the load during the outage. They will not redistribute until they expire naturally.
      • -
    • Failover with a preferred primary: when the primary WAN recovers, existing connections stay on the failover WAN until natural expiry rather than snapping back. In cost-sensitive setups where the primary is cheaper or unmetered, lingering on the failover WAN has a direct cost.
      • +
    • Failover with a preferred primary: when the primary WAN recovers, existing connections stay on the failover WAN until natural expiry rather than snapping back. New connections return to the recovered primary on their own, since the snap-back clears the worse-metric sticky pins so the next new flow re-picks the higher-priority WAN, and where stickiness is not in use new flows re-pick by policy regardless; only established connections linger, which is the case this bullet addresses. In cost-sensitive setups where the primary is cheaper or unmetered, lingering on the failover WAN has a direct cost.
    • Application of new policies / rules to all flows immediately: a config save will cause all new connections be be immediately routed according to the new policy / ruleset, but will leave existing connections intact rather than interrupting them. Users can choose to override that behaviour and make the new policy take effect immediately even for existing connections, but at the cost of interrupting connections such as SSH, file downloads, etc.
      • @@ -1211,7 +1349,7 @@ mwan3_flush_marked_conntrack

      11. Atomic Non-destructive Reload

      reload_service is a single atomic nft -f batch that rebuilds the entire ruleset while the old one is still serving traffic, committing in one kernel transaction with zero traffic disruption window.

      -

      MWAN3_BATCH_DEPTH Counter

      +

      MWAN3_BATCH_DEPTH Counter

      MWAN3_BATCH_DEPTH is an integer counter (default 0) in common.sh. It enables nested batch accumulation:

      • mwan3_nft_batch_start: increments depth; truncates batch file only at depth 0->1.
        • @@ -1219,7 +1357,7 @@ mwan3_flush_marked_conntrack
      • mwan3_nft_exec: routes to mwan3_nft_push when depth > 0, accumulating all operations.

      This means individual build functions (which each call mwan3_nft_batch_start / mwan3_nft_batch_commit internally) work correctly both standalone (immediate kernel commit) and when called from inside a reload batch (operations accumulate, committed as one transaction at the end).

      -

      Preamble (mwan3_nft_reload_start)

      +

      Preamble (mwan3_nft_reload_start)

      Opens the outermost batch (depth 0->1), then writes:

      1. Flush all 8 skeleton chains (mwan3_prerouting, mwan3_output, mwan3_postrouting, mwan3_ifaces_in, mwan3_rules, mwan3_connected, mwan3_custom, mwan3_dynamic).
        2. @@ -1276,7 +1414,9 @@ mwan3_flush_marked_conntrack auto-merge counter } -

        Placing counter inside the flags list causes an nft parse error.

        +

        Placing counter inside the flags list causes an nft parse error. The auto-merge line appears here only because this set has no timeout; a set with a timeout omits it (see auto-merge and timeouts below).

        +

        auto-merge and timeouts

        +

        User-defined sets are declared with flags interval, and auto-merge is appended only when no timeout is configured. auto-merge coalesces any run of contiguous addresses into a single merged range, a useful memory optimization for static CIDR sets, but the merged range carries only one timeout for the whole run. On a timeout-bearing set populated host by host by an external resolver, for example a list domain set filled by dnsmasq nftset= (or smartdns), that coalescing discards the per-element TTLs: under the incremental insertion a resolver uses, the merged range keeps the timeout of whichever element most recently extended it, so a short-lived member extending a run of long-lived ones evicts the whole range early, and a long-lived member extending a run keeps a short-lived member alive well past its TTL. For resolver-populated sets the per-element TTL is the entire point, so auto-merge is dropped whenever a timeout is set. The interval flag is retained either way, so timeout sets can still hold CIDR entries; only the coalescing is dropped.

        dnsmasq Integration

        For each list domain entry, mwan3_write_dnsmasq_fragments writes a dnsmasq confdir fragment:

        nftset=/youtube.com,googlevideo.com/4#inet#mwan3#youtube_v4
        @@ -1288,7 +1428,7 @@ mwan3_flush_marked_conntrack

        mwan3-migrate-ipset-v4.sh (a one-shot script run from postinst and immediately deleted) copies config ipset sections from /etc/config/firewall to /etc/config/mwan3 if any of the sets in /etc/config/firewall are referenced in an mwan3 user rule. The match UCI option required by fw4 is silently ignored by mwan3 (mwan3 does not call config_get match).

        Orphan Cleanup

        mwan3_cleanup_orphaned_ipsets is called in reload_service after mwan3_render_config_ipsets. It queries nft list table inet mwan3 for user-defined sets (those not prefixed mwan3_), compares against currently configured set names, and emits delete set for any orphans. This prevents stale sets accumulating when a set is removed from UCI config via LuCI.

        -

        Rule Integration (ipset and ipset_src)

        +

        Rule Integration (ipset and ipset_src)

        The ipset UCI option on a rule section matches the destination address against a named nft set. The ipset_src option provides the complementary source address match. Both can be set on the same rule and are ANDed together.

        UCI Example

        config rule 'corp_to_wan2'
@@ -1443,6 +1583,7 @@ ubus call mwan3 nftset_elements '{"set":"youtube_v4","max":200}'

        Rule shadowing check: ruleAContainsB() determines whether rule A (earlier) is a superset of rule B (later). The check is deliberately conservative: it only flags clear containment. CIDR containment is computed precisely for both IPv4 (32-bit unsigned arithmetic) and IPv6 (BigInt). Port specs use conservative matching: a rule with no port restriction contains any other; two rules with non-empty port specs are only flagged as contained when the specs are identical strings. When both rules reference an NFT set, the set names are compared: if both rules reference the same nftset the shadowing check proceeds normally; if the set names differ the check is skipped conservatively since set membership cannot be evaluated statically. If only one rule uses a set the check is also skipped. Address family is respected: an IPv4-only rule does not shadow an IPv6-only rule.

        15.1.9 Notify

        The Notify tab (notify.js) provides a raw textarea editor for /etc/mwan3.user. This file is sourced by the mwan3 hotplug handler at the end of interface up/down events, allowing site-specific shell commands to run in response to WAN state changes. The tab reads the file via fs.read on load and writes it back via fs.write on save; no UCI is involved.

        +

        The menu node admin/network/mwan3/notify declares an ACL dependency on luci-app-mwan3-notify, so the tab is absent entirely for a session that does not hold that group. Where the group is held read-only, notify.js sets isReadonlyView from L.hasViewPermission(), which suppresses handleSave and disables the textarea, leaving the hook visible but not editable.

        15.2 Status App

        The Status app provides live monitoring and diagnostic tools. All tabs that show runtime data poll automatically via the LuCI poll mechanism.

        15.2.1 Overview

        @@ -1455,17 +1596,18 @@ ubus call mwan3 nftset_elements '{"set":"youtube_v4","max":200}'

        For the selected interface, the tab shows a header card with status, tracking method, and composite reliability score, followed by a table of individual tracking targets. Each row shows the target address, probe status (up/down/skipped), current latency, and current loss percentage. When check_quality is disabled, the latency and loss columns show "Not enabled". Down targets show an infinity symbol for latency. Rows are sorted: up targets first, then down, then skipped.

        15.2.3 Routing Health

        The Routing Health tab (routing.js) polls mwan3.routing_health and performs a live sanity check of the ip rules and routing tables mwan3 maintains for each configured interface.

        -

        Per-interface cards: Each card shows the interface name, sequential index, and mwan3track-reported status, followed by four status rows:

        +

        Per-interface cards: Each card shows the interface name, sequential index, and mwan3track-reported status, followed by four status rows, five on IPv4 interfaces:

        • IP rule (iif) at priority iif_base + index: expected present when the interface is online.
        • IP rule (fwmark) at priority fwmark_base + index: expected present when the interface is online.
        • IP rule (unreachable) at priority unreachable_base + index: expected present when the interface is online.
          • +
        • IP rule (source) at priority src_base + index: expected present when the interface is online. Rendered only for IPv4-family interfaces, the only ones for which routing_health reports a src_rule object. A device holding several global IPv4 addresses carries one rule per address at this priority.
        • Routing table (table ID = index): whether the per-interface routing table has a default route.
        -

        Each row renders a status badge: "Present" (green) or "Missing" (red) when the item is expected to be present (interface online); "Present (unexpected)" (yellow) or "Absent" (grey) when expected to be absent (interface offline); neutral grey "Present"/"Absent" for unknown or disabled interfaces. The card border is green when all four items are in the expected state, yellow for partial state, red when the interface is online and items are missing, and grey for offline or disabled interfaces in the expected state.

        +

        Each row renders a status badge: "Present" (green) or "Missing" (red) when the item is expected to be present (interface online); neutral grey "Present"/"Absent" for every other state, where presence is reported without judgement. The card border is green when all rows are in the expected state, yellow for partial state or for an interface reported offline or disconnecting, red when the interface is online and no rules are present, and grey for disabled or unknown interfaces.

        Summary card: A single line at the top reports total healthy, degraded, and failing interface counts, plus stale rule count if any.

        Stale rules: ip rules whose priority falls within the mwan3 priority ranges but do not correspond to any currently configured UCI interface are listed in a separate table. Stale rules indicate that mwan3 was not cleanly shut down after an interface was removed from the configuration.

        -

        Field guide: A reference panel at the bottom explains each field (Index, iif rule, fwmark rule, unreachable rule) in plain language, using the actual configured base priority values read from the routing_health response rather than hardcoded defaults.

        +

        Field guide: A reference panel at the bottom explains each field (Index, iif rule, fwmark rule, unreachable rule, source rule) in plain language, using the actual configured base priority values read from the routing_health response rather than hardcoded defaults. The source base is derived by the backend as unreachable_base + iface_max + 1 rather than read from UCI; the fallback in mwan3/constants.js is 3061, the value the default bases and the default 0x3F00 mask imply.

        15.2.4 IP Sets (Status)

        The IP Sets status tab (ipsets.js) displays runtime information about all user-defined sets currently present in table inet mwan3. At page load it calls mwan3.nftset_info to enumerate sets and their metadata, then renders each as a collapsible panel.

        Panel header: Shows the set name, family, element count (with N+ suffix when the display is truncated), a Counters badge when per-element counters are enabled, and action buttons: Resolve (only for sets with at least one domain entry), Reload, Flush, and Expand/Collapse.

        @@ -1500,9 +1642,10 @@ ubus call mwan3 nftset_elements '{"set":"youtube_v4","max":200}'

        nft list sets inet mwan3 is not used because it lists sets from all inet tables, not just mwan3's table. nft list table inet mwan3 is the correct form.

    ACL Permissions (luci-app-mwan3.json)

    -

    The rpcd ACL file grants the LuCI frontend permission to call rpcd methods and execute files. It defines two ACL groups:

    -

    luci-app-mwan3 (configuration access): Read permission for mwan3.status, mwan3.nftset_info, mwan3.nftset_members, mwan3.nftset_elements, mwan3.resolve_host, and the mwan3 and network UCI packages. Write permission for mwan3.nftset_flush, mwan3.nftset_reload, mwan3.nftset_resolve, and the mwan3 UCI package. Also grants fs.stat read access for detecting optional binaries (httping, nping, arping) and exec permission for luci-mwan3 nftset dump.

    -

    luci-app-mwan3-status (status and diagnostic access): Read permission for mwan3.status, mwan3.nftset_info, mwan3.nftset_members, mwan3.nftset_elements, mwan3.resolve_host, and mwan3.routing_health. Write permission for mwan3.nftset_flush, mwan3.nftset_reload, mwan3.nftset_resolve, and exec permission for the luci-mwan3 diag subcommands, mwan3 internal, mwan3 ifup, and mwan3 ifdown.

    +

    The rpcd ACL file grants the LuCI frontend permission to call rpcd methods and execute files. It defines three ACL groups:

    +

    luci-app-mwan3 (configuration access): Read permission for mwan3.status, mwan3.nftset_info, mwan3.nftset_members, mwan3.nftset_elements, mwan3.resolve_host, and the mwan3 and network UCI packages. Write permission for mwan3.nftset_flush, mwan3.nftset_reload, mwan3.nftset_resolve, and the mwan3 UCI package. Optional-binary detection needs two grants working together: the list permission on /usr/bin/httping, /usr/bin/nping and /usr/bin/arping, which is what rpcd's stat handler checks on the path, and the stat method on the ubus file object, which the HTTP ubus endpoint checks before dispatching the call. The read block also grants cgi-io exec and exec permission for luci-mwan3 nftset dump.

    +

    luci-app-mwan3-status (status and diagnostic access): Read permission for mwan3.status, mwan3.nftset_info, mwan3.nftset_members, mwan3.nftset_elements, mwan3.resolve_host, and mwan3.routing_health, together with cgi-io exec and exec permission for /usr/sbin/mwan3 status. Write permission for mwan3.nftset_flush, mwan3.nftset_reload, mwan3.nftset_resolve, the exec method on the ubus file object, and exec permission for the luci-mwan3 diag subcommands, mwan3 internal ipv4 and mwan3 internal ipv6, mwan3 ifup, and mwan3 ifdown.

    +

    luci-app-mwan3-notify (hotplug hook editing): Read permission for /etc/mwan3.user together with the read method on the ubus file object. Write permission for the same path together with that object's write and remove methods. The group is separate so that delegating mwan3 configuration does not imply the ability to edit the hook: /etc/mwan3.user is a shell script the hotplug handler runs as root on every interface event, so write access to it is equivalent to root. Before the split these grants sat in the configuration group. A session without the group does not see the Notify tab, read gives a read-only view, and write gives editing.

    15.4 rpcd Methods

    The following methods are implemented in usr/share/rpcd/ucode/mwan3 and declared in root/usr/share/rpcd/acl.d/luci-app-mwan3.json.

    @@ -1531,7 +1674,7 @@ ubus call mwan3 nftset_elements '{"set":"youtube_v4","max":200}'

    mwan3.routing_health {}

    Compares the UCI configuration against live kernel state. For each mwan3 interface (by 1-based UCI order index N):

      -
    • Checks for ip rule at priority IIF_BASE+N (iif), FWMARK_BASE+N (fwmark), and UNREACHABLE_BASE+N (unreachable) via rtnl RTM_GETRULE. Base values read from UCI globals, defaulting to 1000/2000/3000.
      • +
    • Checks for ip rule at priority IIF_BASE+N (iif), FWMARK_BASE+N (fwmark), UNREACHABLE_BASE+N (unreachable), and SRC_BASE+N (source, IPv4 interfaces only) via rtnl RTM_GETRULE. Base values read from UCI globals, defaulting to 1000/2000/3000, with SRC_BASE derived as UNREACHABLE_BASE + iface_max + 1.
    • Checks routing table N for a default route via rtnl RTM_GETROUTE
    • Reads /var/run/mwan3track/<ifname>/STATUS for current online/offline state
    • Reports stale ip rules (priorities in mwan3's range with no matching UCI interface)
      • @@ -1604,7 +1747,189 @@ mwan3-lb-test cleanup

      Files changed: usr/sbin/mwan3-diag (new), Makefile

      18. Changelog

      -

      18.1 Version 3.6.9

      +

      18.1 Version 3.6.12

      +

      Summary: Adds IPv4 source-address return routing. Router-originated traffic sourced from a WAN's own address must leave by that WAN, but a sender that routes itself outside netfilter's view followed the main table out the default WAN and was dropped upstream or masqueraded to the wrong address. WireGuard is the case that motivates it, where replies to a handshake arriving on a non-default WAN left by the default WAN and the handshake never completed. One ip rule per global-scope address on each WAN now sends those lookups back to the arrival WAN. The rpcd status method reports the new rules and the web interface shows them alongside the existing three on its routing health view.

      +

      A route wide enough to carry the whole internet without being a literal default is now treated as part of a default route when the bypass sets are built. VPN clients routinely install exactly that, OpenVPN with redirect-gateway def1 and split-tunnel WireGuard installing a pair of half-width routes rather than a default, and some IPv6 providers pushing the global-unicast prefix. One such route anywhere mwan3 scanned handed the entire address family to the bypass and silently stopped rule steering, sticky pinning, blackholing and inbound reply-path classification from working. Routes in the extra lookup tables are also no longer recorded in the connected-networks set, where a stale copy could outlive the route itself and keep a destination exempt from policy routing for hours.

      +

      Three faults in rule building are corrected. An nft set referenced by rules of both address families but defined nowhere in the config was created twice with conflicting types, which nft rejected, leaving the router with no user rules at all. A sticky rule matching both families built its per-rule chain with IPv4 lookups only, so IPv6 flows were never pinned and stickiness silently applied to one family, and for ICMP the late IPv6 lookup produced a mark matching no interface at all. IPv6 policy members are no longer added on kernels without IPv6, where a policy with no usable IPv4 members steered IPv4 traffic to an IPv6 member instead of falling through to its last resort action.

      +

      Three further faults affect the service itself. A service that started with every interface disabled never created its nft table, so a later config change reloaded against nothing and left the router with no ruleset until a manual restart, and now falls back to a full restart instead. The two route monitor instances shared a single nftables batch file and could race on it around WAN transitions and prefix renewals, leaving the bypass sets stale, and each now has its own. The route seeding helper counted an interface that omits the enabled option as enabled, where every other consumer of an interface section treats it as disabled, which withheld that device's routes from every other interface's table.

      +

      The diagnostic report no longer prints public addresses that appear only in the nft ruleset or the log, service stop time is reduced by dropping tear-down code left over from the fw4 era, and the shipped base files now link to the nftables documentation rather than the legacy iptables page. In the web interface, address lists in table columns are reduced to their first entry and a count of the remainder, so a long comma-separated list no longer stretches or overflows the cell, and the address checks now call the framework's own parsers.

      +
      + +

      The default UCI config and the hotplug script both linked to the legacy iptables wiki page. Point them at the nftables page, whose sections describe the configuration options and hotplug environment that this package actually implements.

      +
      +

      mwan3: keep extra lookup table routes out of the connected set

      +

      Destinations with a route in one of the rt_table_lookup tables are exempt from policy routing. The route monitor recorded these routes in two places: the set that tracks them and the set that tracks directly connected networks. The first is updated whenever a route in those tables is added or deleted, but the second is only rebuilt when the main routing table changes. Deleting a route from an extra table therefore ended its exemption in one set while the copy in the other kept it alive, and traffic to that destination continued to bypass policy routing, for hours or days on a quiet router.

      +

      Record only main table routes in the connected networks set, as the startup code already does. Nothing is lost by this: the set dedicated to the extra table routes grants the same exemption and is kept current.

      +
      +

      mwan3: keep default-equivalent routes out of the bypass sets

      +

      Destinations that mwan3 should not steer are collected into bypass sets built by scanning the routing tables, and anything in those sets is marked for the main table before and after the user rules run. The scan recognises a default route only in its literal form, so a route wide enough to carry the whole internet without actually being a default looks like an ordinary network to it. VPN clients routinely install exactly that: OpenVPN with redirect-gateway def1 and split-tunnel WireGuard install a pair of half-width routes and not a default, and some IPv6 providers push the global-unicast prefix. One such route anywhere mwan3 scans hands the entire address family to the bypass, and rule steering, sticky pinning, a blackhole and inbound reply-path classification stop working silently.

      +

      Consider a route at or below /2 for IPv4 and /3 for IPv6, or with no destination at all, to be a component of a default route. Nothing a real network hands out is that wide. Anyone who genuinely wants a broad bypass still has the bypass_network option. Routes are judged one at a time, so a client that has installed only half its pair is covered too.

      +

      Only the bypass sets use the wider definition. Filling an interface's own table with routes, and taking that interface offline when its table holds no default both still require a literal default route.

      +
      +

      mwan3: fix interface enabled in route seeding helper

      +

      The interface route seeding helper read an interface section's enabled option with a default of 1, where every other consumer of an interface section defaults it to 0. The init script, the hotplug handler, the interface state paths and the route monitor all treat a section that omits the option as disabled.

      +

      The defect is longstanding: the shell function this helper replaced called config_get_bool without its default argument, so a section with no enabled option left the variable empty and the numeric test meant to skip the interface failed on an empty string rather than skipping it.

      +

      Counting the interface as enabled put its device into the device-to-table map, which withheld that device's routes from every other interface's table while delivering them to a table that is never built, because mwan3 never brings up a disabled interface. Nothing relied on that.

      +

      The visible consequence was that traffic in another interface's table addressed to a subnet directly connected to that device found no specific route and fell through to that table's default route.

      +
      +

      mwan3: fix public addresses leaking into the mwan3-diag report

      +

      Some public addresses were printed unmasked. The anonymiser replaces only addresses learned from neighbours, routes, rules, interface addresses, status and config, so an address appearing only in the nft ruleset or the log, such as a NAT map element, was printed as-is.

      +

      Now the nft and log text the report prints is also swept for addresses before placeholders are assigned, so every public address in the report is masked.

      +
      +

      mwan3: give each mwan3rtmon instance its own nft batch file

      +

      mwan3rtmon runs as two long-lived instances, one per address family, and both wrote their nftables batches to the same fixed-name file, opened with truncation and never removed. When both families' routes changed at the same time, as is common around WAN transitions and prefix renewals, the two instances could race on the shared file: one could truncate or rewrite it while the other was writing or loading it with nft -f, so a load could apply a truncated, mixed or wrong-family batch, or fail outright, leaving the connected or custom destination sets stale until the next route change rebuilt them. The last batch file was also left behind after every load.

      +

      Name the batch file with the instance's address family, which is sufficient because one instance runs per family and each instance is single-threaded, and remove the file on every exit path after it is created, matching the per-process, self-cleaning convention of the shell batch machinery in common.sh.

      +
      +

      mwan3: create a missing set only once per rebuild

      +

      A rule can match on an nft set that mwan3 does not define in its own config, leaving it to be created in table inet mwan3 by something else, or never created at all through a config error. mwan3 creates an empty set when it finds one missing, as an unresolved set reference would break the atomic nft -f batch load.

      +

      The address type came from whichever rule needed the set, so two rules matching the same missing set from different families each created it, with conflicting types. nft rejected the batch, leaving the router with no user rules at all.

      +

      mwan3 now remembers the sets it creates as it builds. The first rule to need a missing set settles its address type, while a later rule of the other family is skipped, quietly if it is a family=any rule, or with a warning if the rule is family specific.

      +
      +

      mwan3: skip IPv6 policy members on kernels without IPv6

      +

      Ensure that IPv6 policy members are only added when the kernel supports IPv6. Such members previously bypassed member selection but were still added to the chain, so a policy with no usable IPv4 members steered IPv4 traffic to an IPv6 member instead of falling through to its last_resort action.

      +
      +

      mwan3: complete sticky chains for family=any rules

      +

      A sticky rule with family=any and no family-specific match element is rendered once, as a single line matching both address families, but the per-rule sticky chain it jumps to was built with IPv4 sets, lookups and updates only. IPv6 flows matching the rule entered the chain, missed every lookup and were never pinned: the policy re-evaluated each new flow, so connections from one source could egress different WANs. Stickiness silently applied to IPv4 only.

      +

      For proto=icmp such a rule renders on both family passes and each pass appended its own family's sticky arm to the shared chain. The IPv6 lookups landed after the IPv4 arm's fall-through, so pinned IPv6 sources were re-marked by the policy first and the late lookup then ORed the old pin into the new mark, producing a value matching no interface.

      +

      Fix both by building the per-rule chain exactly once, complete for every family whose traffic can reach it: all lookups, then the single fall-through to the policy, then all update rules. Rules with an explicit family or a family-specific match element render as before.

      +
      +

      mwan3: report IPv4 source rules in routing_health

      +

      The routing_health method reported the iif, fwmark and unreachable ip rules per interface but not the IPv4 source-address return routing rules, so a consumer could neither display them nor factor them into an interface's health.

      +

      For each IPv4-family interface, add a src_rule object carrying the rule's presence and priority (unreachable base + interface maximum + 1 + index), and extend rule_bases with the source base. Presence is tested against a family-filtered index of the IPv4 rule dump so an unrelated IPv6 rule sharing the priority is not counted. A device with several global IPv4 addresses holds one rule per address at this single priority, so presence means the slot is populated rather than that every address is covered. IPv6-family interfaces carry no src_rule key.

      +
      +

      mwan3: fix whitespace in rpcd module

      +

      Adjust blank-line spacing around several comment blocks in the rpcd module for consistency with comment formatting elsewhere, and remove one blank line between two variable declarations.

      +
      +

      mwan3: restart on reload when the nft table is missing

      +

      When the service starts with every interface disabled, start_service returns before creating the nft table, yet the status directory makes it report running, so a config change drives a reload that rebuilds against the missing table. The batch fails and mwan3 is left with no ruleset until a manual restart.

      +

      Guard reload_service with a table-existence check and fall back to a full restart when the table is absent.

      +
      +

      mwan3: optimise service stop time

      +

      Optimise mwan3 service stop time by removing table inet fw4-era manual tear-down code.

      +
      +

      mwan3: add IPv4 source-address return routing

      +

      Router-originated IPv4 traffic sourced from a WAN's own address must leave by that WAN, but a sender that routes itself outside netfilter's view follows the main table out the default WAN and is dropped upstream or masqueraded to the wrong address.

      +

      WireGuard is a good example: replies to a handshake arriving on a non-default WAN leave by the default WAN and the handshake never completes. Locally originated flows that no user rule classifies share the gap.

      +

      Install one ip rule per global-scope IPv4 address on each WAN, from <addr> lookup <table>, on the same lifecycle as the interface's existing rules. Route lookups then resolve back to the arrival WAN and the reply keeps the correct source. Marked traffic still outranks these rules, so policy verdicts and the fail-closed backstops behave as before, and the rules are inert for forwarded traffic and single-WAN setups.

      +

      Update the rpcd module to recognise the new source rules, which its routing health check would otherwise report as stale.

      +
      +

      luci-app-mwan3: fix ACL grant for optional binary detection

      +

      Detection of the optional tracking binaries runs through fs.stat, whose ubus method was not granted by the configuration group. Sessions holding only this app's ACL groups therefore offered ping as the only tracking method even with the binaries installed. Grant the stat method.

      +
      +

      luci-app-mwan3: replace custom validator calls with framework calls

      +

      The rule and IP set address checks ran the ipaddr datatype through a custom validator object.

      +

      Add ipAddrOrPrefix() and ipv6Cidr() to the validators module, both built on the framework's parseIPv4() and parseIPv6(). The three address checks now call ipAddrOrPrefix(). The custom validator stays for the types the framework publishes no parser for.

      +
      +

      luci-app-mwan3: summarise address lists in table columns

      +

      The source and destination address options accept a comma-separated list, and every table that previews a rule rendered that list in full. A comma-joined list is a single long token with nothing to break on, so it stretches an auto-layout table column and overflows the cell of a fixed-layout one.

      +

      Add a shared formatter that reduces such a list to its first entry followed by a count of the remainder. The stored value is untouched and the editing widgets still show every entry, so the complete list remains available in the rule modal.

      +

      The source preview is reshaped to build its text from a list of parts, matching the destination preview directly below it, which already does so because it appends a mark element.

      +

      The translation template is regenerated. The only changes are moved source line references, not strings.

      +
      +

      luci-app-mwan3: regenerate translation template

      +

      The translation template had not been regenerated since the last change that added strings to it. Regenerate it from current sources. No functional change.

      +
      +

      luci-app-mwan3: show IPv4 source rules on Routing Health

      +

      The routing_health backend now reports the IPv4 source-address return routing rules, but the Routing Health tab rendered only the iif, fwmark and unreachable rows and ignored the source band.

      +

      Render a fourth "IP rule (source)" row on IPv4-family interface cards, keyed off the src_rule object so it never appears where the concept does not apply, and fold the source rule into the online health judgement so a missing rule degrades the card. Document the rule in the field reference, covering its purpose, lifecycle, the one-rule-per-address behaviour and its IPv4-only scope, and update the closing summary for the fourth row. Add a src fallback to the shared rule-base defaults for when the backend does not report bases.

      +
      +

      18.2 Version 3.6.11

      +

      Summary: Fix the nft rule rendering order such that it preserves strict config order.

      +
      +

      mwan3: render user rules in config order

      +

      The user rule renderer looped over address family first and the config rules second, so the installed mwan3_rules chain held the entire IPv4 render pass followed by the entire IPv6 render pass. Config order was preserved within each pass but not across the pass boundary.

      +

      A family=any rule with no IP-version-specific match element (for example a bare protocol and port rule, or an fwmark rule) is emitted once, on the IPv4 pass, as a single family-agnostic line that also matches IPv6 traffic at runtime. Because the whole IPv4 pass rendered first, that line always landed ahead of every IPv6-only rule in the chain regardless of its position in the config, and shadowed any IPv6 rule the configuration placed before it. Rule evaluation is first match, so no config ordering could place an IPv6-only rule ahead of a family-agnostic one, and reordering rules in the config silently failed to change evaluation order.

      +

      Invert the loop nesting: iterate the config rules in order and render each rule's IPv4 and IPv6 variants at that rule's position. The per-rule, per-family emission function is unchanged, so every rendered line is identical; only the order of the lines in mwan3_rules changes, and it now always matches config order. The chain is a flat first-match list with no structural dependency on the old pass grouping, and per-rule sticky chains and sets are keyed by rule and family, so their content is unaffected.

      +

      For configurations that interleave family-agnostic rules with family-specific ones, evaluation order now follows config order strictly.

      +
      +

      18.3 Version 3.6.10

      +

      Summary: This release corrects six longstanding bugs, all present in the original codebase, fixes some privilege escalation issues, corrects a tracking-health edge-case, modularises and refactors the LuCI app for maintainability and to make exclusive use of LuCI's published API, and makes ucode scripts log to syslog. Normal operation while every internet connection is healthy is unchanged throughout.

      +

      Four of the changes address what happens at the moment a connection fails, recovers, or is brought up or down. When a connection was detected as failed while still physically connected, the records of which sessions and devices were using it were not cleared, so traffic stayed pointed at the dead connection for minutes instead of moving to a working one.

      +

      When a preferred connection later recovered, traffic that had diverted to a backup did not return to it. Both are now corrected: on failure the records are cleared immediately, and on recovery new traffic returns to the better connection while sessions already in progress finish where they are.

      +

      The order in which routing and connection selection are updated during a transition was also wrong in both directions: on recovery, a connection was made available for traffic before its route was rebuilt, so traffic was assigned to it before it could carry any. On shutdown, a connection's route was removed before traffic stopped being assigned to it. Both orders are now correct.

      +

      A per-connection safety block that rejects traffic with no working route was being removed when a connection went down, at the exact moment it was needed. It is now kept in place for as long as the service runs, so traffic with no route is rejected rather than escaping over the default internet link.

      +

      All forwarding-rule changes during a transition are now applied as a single all-or-nothing update, closing a brief window in which the rules were only partly in place and traffic could be steered by an incomplete set of them.

      +

      A separate fix removes a fallback rule that silently ran in place of a policy's configured behaviour for when all its connections are down, sending traffic out the default link even when the policy was explicitly set to block it.

      +

      The sixth fix corrects corruption of per-entry expiry times in address sets populated by a DNS resolver: a kernel range-merging optimisation was discarding the individual expiry of each address and replacing it with the expiry of whichever address had last extended the merged range, causing entries to expire too early or be kept alive too long.

      +

      Three further fixes harden the service against a user who has been granted only limited, delegated control of the mwan3 configuration, closing routes by which that restricted role could take over the whole router or another user's web session. Two of them address ways such a user could gain administrator-level command execution: one web-interface permission let them edit a script that mwan3 runs as the administrator on every connection event, and two internal helpers built system commands out of configuration values, such as a domain name or an address-set entry, without first checking them. That editing permission is now separated into its own group that ordinary configuration access no longer carries, and every value is checked before it reaches a command. The third fix escapes configuration text before it appears on the status pages, so text saved by one user can no longer run as code in the browser of anyone who later views those pages.

      +

      Several further, unrelated changes round out the 3.6.10 release. A missing default route on an interface's own routing table is now treated as a tracking failure, so an interface that its tracker would otherwise still report as online is taken offline instead of blackholing the traffic steered to it. The routing-health view in the web interface has been updated to match current behaviour, in which the per-interface reject rule is a permanent fixture rather than one that appears and disappears with the interface, so its presence on an offline interface is now shown as expected rather than as a fault. The web interface itself has also been reworked to build on LuCI's standard, supported components and split into shared modules for easier maintenance, with no change to how it behaves aside from a few small display corrections. Finally, the ucode components now log to the system log with a per-component tag, matching the shell components.

      +
      +

      mwan3: validate ipset elements and resolver hosts before use

      +

      The rpcd backend runs as root. Two of its methods built command lines from UCI-derived values with no validation and ran them through popen, executing via a shell fork. nftset_resolve passed each domain straight to the nslookup command line and nftset_reload joined the section's entry values and the lines of a loadfile into an nft add element command line. A user holding only a delegated config role has UCI mwan3 write plus these methods, so a crafted domain, entry, or loadfile line could potentially run arbitrary commands as root.

      +

      The same entry list is consumed a second time by the ordinary ipset apply path, which builds an add element batch that is later committed with nft -f. There a crafted value could close the element braces early and inject rogue nftables statements, executed as root on the next apply.

      +

      Reject anything that is not a plain hostname before it reaches nslookup. Validate every ipset element, from both the entry list and the loadfile, ensuring it is a valid IPv4 or IPv6 address, a CIDR, or a hyphen range, and drop anything else before it reaches a command line. Apply the element check both in the ucode and the shell path.

      +
      +

      mwan3: log ucode components to syslog

      +

      Route the ucode logging through a new shared logging function housed in common.uc that writes to syslog with a per-component tag, just like the shell components do via the logger tool.

      +

      Add a ucode-mod-log dependency.

      +
      +

      mwan3: take an interface offline when its table has no default route

      +

      An interface's tracker reports it online on the strength of its probes, which reach their target over a connected or point-to-point route and do not need a default route. So an interface whose own routing table holds no default route still reports online: mwan3 marks traffic to it, its fwmark ip rule sends that traffic to the interface's table, the table has no default, and the per-interface unreachable backstop drops it. The interface black-holes every flow steered to it while still showing online.

      +

      Make a missing default route count as a tracking failure. The route monitor, which already maintains each interface's own table, checks whether that table holds a default route via the interface's device and writes up or down to a small file in the tracker's status directory. The tracker reads it once per round and, when it reads down, skips the probe so the round fails; the existing score and hysteresis then carry the interface offline, and back online when the route returns, with no new state and no new hotplug arm. The check is on the interface's own table, not the main table, because the fwmark rule looks up exactly that table, and it is recomputed from a full route dump rather than a single route event, so an interface that never held a default also reads down. An absent file is treated as up, so a tracker with no route monitor is unaffected, and the unreachable backstop is unchanged, so traffic still marked to a downed interface fails closed rather than leaking.

      +
      +

      mwan3: batch each failover transition into one nft transaction

      +

      When a WAN's usability changes, whether it stops being usable (a tracker-detected soft failure or an administrative or carrier down) or becomes usable again (a tracker-detected recovery, or coming back up while already online), mwan3 rebuilds its routing policy, and on a teardown also clears the per-source stickiness records that kept traffic on the failed WAN. Until now mwan3 issued these as several separate firewall operations.

      +

      Because they were separate, the firewall could be observed in an intermediate state during the transition: a packet could be classified against a policy that had been flushed but not yet fully rebuilt, and on a teardown the stickiness records were cleared in a step that could briefly disagree with the rebuilt policy. These windows are sub-second, but they are real and serve no purpose.

      +

      Apply the policy rebuild as a single firewall transaction on every transition, both the teardown paths and the recovery paths, committed in one step. The kernel applies the whole transaction at once, so traffic is never steered by a half-built policy. The resulting ruleset is identical to what the separate operations produced; only the number of kernel transactions changes.

      +

      On a teardown the stickiness clearing joins the same transaction, in the fixed order of the policy first and the stickiness second, because the stale stickiness points at a WAN that is going away and must be cleared in the same step that drops it from the policy. On a recovery the stickiness reset that returns new traffic to the recovered WAN stays a separate step after the rebuild: it only resets records pointing at lower-priority WANs that are still working, so a connection that consults one in the instant before the reset still reaches a working WAN, and its timing is not safety-critical.

      +

      The connection-tracking cleanup stays a separate step at the end of the transition, because connection-tracking marks are kernel state outside the firewall ruleset and cannot share its transaction.

      +

      This corrects longstanding behaviour rather than a regression; the separate-operation form predates mwan3's use of atomic firewall transactions.

      +
      +

      mwan3: install static per-WAN unreachable backstops

      +

      mwan3 sends a connection out a chosen WAN by tagging it with a firewall mark and adding an ip rule that routes that mark to the WAN's table. Next to each of those it also installs a backstop rule that rejects the mark as unreachable when the WAN has no usable route. That backstop is what keeps traffic tagged for a WAN from quietly slipping out the main routing table, and the default route, when the WAN cannot carry it.

      +

      The bug was that the backstop was added and removed together with the WAN's routing rules, so it only existed while the WAN was up. It was therefore missing at the worst possible moment: right after a WAN goes down, or during the split second its rules are torn down and rebuilt, a connection still tagged for that WAN had nothing to reject it and leaked out the default route. On a policy meant to be a hard boundary, such as a VPN, that is a genuine leak.

      +

      The fix makes the backstops permanent. mwan3 now installs one for every configured WAN when the service starts and on every reload, and removes them only when the service stops. The rules that actually route into a WAN still come and go with it, because they need the live interface, but the reject rule stays put. Traffic tagged for a WAN with no matching route, whether the WAN is down, disabled, or briefly mid-rebuild, is now rejected instead of leaking. This covers every configured WAN, including disabled and offline ones, since a WAN disabled while its connections are still live leaves tagged traffic that has to be caught.

      +

      The backstops use only real WAN marks, never mwan3's reserved marks, so a policy's last_resort setting is untouched: last_resort "default" still deliberately uses the main routing table, "unreachable" still rejects, and "blackhole" still drops. The new rules are created through the same helper that already builds mwan3's global rules, and the existing stop-time cleanup already removes them. This corrects long-standing behaviour carried over from the old iptables-based mwan3; it is not a regression.

      +
      +

      mwan3: clear stale WAN pins on soft failover and snap back on recovery

      +

      When a WAN goes up or down, mwan3 has to bring two caches back into step with the routing policy it rebuilds. Both caches decide a packet's WAN before the policy is consulted, and both can override it: the sticky sets, which tie a source address to a WAN without checking that the WAN is still up, and the connection marks kept by conntrack, which tie an existing connection to the WAN it started on. Both refresh themselves as traffic flows, so once either points at a WAN the policy has dropped, the mistake lasts for minutes rather than for the moment of the change.

      +

      On a soft failure, where the tracker judges a WAN dead but the interface itself stays up, mwan3 dropped the WAN from the policy but cleared neither cache. A source pinned in the dead WAN's sticky set was still tagged for it, and a connection already marked for it was still restored to it, so traffic either black-holed on the dead WAN or, when that WAN's routing entry was already gone, escaped out the main routing table's default route. Every leaked packet re-armed the sticky timer, so the wrong state held itself open. A hard interface-down cleared the affected connection marks; the soft failure cleared nothing.

      +

      Now both caches are cleared on the soft-failure event, after the policy is rebuilt, in the order a teardown needs: policy first, then the sticky sets, then the connection marks. A sticky entry can stamp its WAN onto a connection's mark, but a connection's mark never feeds back into the sticky sets, so clearing in this order leaves nothing already-cleared to be re-dirtied. The targeted connection-mark flush that a hard interface-down already performed is now applied to the soft failure too. A pinned source picks the survivor on its next new connection, and an existing connection takes one reset and reconnects on the survivor instead of hanging until the dead WAN returns. Protocols that can move a live connection by themselves, such as QUIC, carry on without a reset.

      +

      On recovery, new traffic is moved back off the backups the returning WAN outranks. mwan3 ranks a policy's WANs by metric, lower winning, and keeps IPv4 and IPv6 as two separate contests, so a recovering WAN competes only with others of its own address family. When a higher-ranked WAN comes back, the sources parked on a lower-ranked backup of the same family should return to it. For each policy the recovered WAN belongs to, mwan3 clears the sticky sets of that policy's lower-ranked, same-family WANs, and only for the rules that use the policy, so a parked source re-picks on its next new connection while connections already running on a backup are left alone. The narrowing matters in two ways. Keeping to the WAN's own family means an IPv4 recovery never disturbs IPv6 stickiness, or the reverse. Keeping to the policy and its rules means a WAN that is a backup in a failover policy and an equal-ranked WAN in a separate load-balancing policy keeps its load-balancing stickiness, which clearing by WAN alone would wrongly wipe. In a load-balancing policy every WAN ranks equally, so nothing is cleared and steady-state stickiness is kept.

      +

      These are longstanding behaviours carried over from the old iptables-based mwan3: the sticky scheme never checked whether a WAN was still up, and the soft-failure path never cleared either cache.

      +
      +

      mwan3: restore a WAN's routing on recovery and rebuild policy before teardown

      +

      When a WAN changes state, mwan3 has to update both the routing policy and that WAN's own ip rules and route. The handler that does this updated them in an order that could leave a packet tagged for a WAN whose routing was not in the matching state.

      +

      When the tracker brought a WAN back up, the handler rebuilt the policy but, unlike a normal interface bring-up, never recreated that WAN's own routing rules and route. A WAN restored this way was added back to the policy and had traffic tagged for it, yet nothing routed that traffic until the next reload, so it was misrouted or dropped. The recovery path now recreates the WAN's rules and route before rebuilding the policy, the same as a normal bring-up does, so a recovered WAN can carry traffic the instant it is tagged for. Re-creating the rules is harmless when they already exist, so a recovery that races a WAN that is already up just re-asserts the same rules.

      +

      When a WAN was taken down, by an administrator or by losing carrier, the handler removed that WAN's routing and cleared its sticky sets before rebuilding the policy. In the gap between removing the routing and rebuilding the policy, the still-stale policy could tag traffic for the downed WAN, whose routing rule was already gone, and a packet crossing that gap re-filled the sticky set that had just been cleared. The down path is reordered so the policy is rebuilt first, dropping the WAN before its routing is removed, and the sticky sets are cleared after the rebuild, so the rebuild can no longer re-fill a set it just cleared.

      +

      Both behaviours are longstanding, carried over from the old iptables-based mwan3. Steady-state routing and the built ruleset are unchanged; the difference shows only across a change of state.

      +
      +

      mwan3: remove offline-WAN fallback that overrode last_resort

      +

      When a policy had no WANs online, the policy builder added an extra rule just ahead of the last_resort rule, as part of a broader link-selection change. It matched traffic that would have left through an offline WAN's device and tagged it for the main routing table. Because it sat before the last_resort rule, and both act only while the firewall mark is still unset, it ran instead of last_resort for that traffic.

      +

      This silently overrode a deliberate setting. An operator who sets last_resort to unreachable or blackhole is telling the policy to fail closed when none of its WANs is available. The fallback instead sent some of that traffic out the main table. It picked that traffic by which device it would leave through, not by where it was going, so during a soft failure, where the failed WAN's device is still present and its default route is still in the main table, it also caught ordinary default-routed traffic, not just traffic to directly reachable hosts, and sent it back out the dead WAN via the main table. For a one-WAN policy used as a kill switch, such as a router-originated VPN tunnel with last_resort unreachable, this defeated the fail-closed behaviour for exactly the traffic the operator meant to block.

      +

      Removing the fallback does not change reachability of directly connected or local destinations. Those are sent to the main routing table by the connected-destination and custom-destination checks that run before the policy, so that traffic never reaches the policy and never relied on this rule. The connected set is built from the main table's non-default network routes, so it already covers directly connected subnets and other non-default routes. The fallback's only remaining effect was on default-routed traffic leaving through a fully-down policy's offline WAN, which is exactly the case that should obey last_resort.

      +

      The change is narrow. With last_resort default the policy already tags all its traffic for the main routing table, so nothing changes. Only last_resort unreachable and blackhole are affected, and only for default-routed traffic leaving a fully-down policy's offline WAN, which now fails closed as set. A destination reachable only by a host route out the offline WAN is not in the connected set and now obeys last_resort too; deliberate exceptions are still available through the custom-destination set.

      +

      The now-unused offline-device bookkeeping in the per-WAN policy walk is removed with it.

      +

      Fixes: 46ed09408e55 ("mwan3: refine link selection")

      +
      +

      mwan3: disable auto-merge on timeout-bearing config ipsets

      +

      Config ipset sections are declared as interval sets and auto-merge has been applied unconditionally. When such a set also carries per-element timeouts and is populated host by host by an external resolver, for example dnsmasq nftset= or smartdns, auto-merge coalesces any run of contiguous host addresses into a single range. The merged range carries one timeout and the per-element timeouts of the other members are discarded. Under the incremental insertion a resolver uses, the surviving timeout is that of whichever element most recently extended the range: a short-lived member that extends a run of long-lived members evicts the whole range early when its TTL fires, and a long-lived member that extends a run keeps a short-lived member alive well past its TTL. For resolver-populated sets the per-element TTL is the entire point, so this silently corrupts membership.

      +

      auto-merge remains a useful memory optimization for static CIDR sets, so restrict it to sets without a timeout. The interval flag is retained so timeout sets can still hold CIDR entries; only the auto-merge coalescing is dropped.

      +
      +

      luci-app-mwan3: move mwan3.user editing to its own ACL group

      +

      The configuration ACL group granted read and write on /etc/mwan3.user, along with the ubus file read, write and remove methods. That file is a shell script run as root on every mwan3 hotplug event, so anyone delegated the configuration role could edit it and run arbitrary code as root. That role therefore implied full root.

      +

      Move the /etc/mwan3.user grants and the ubus file read, write and remove methods into a new luci-app-mwan3-notify group, and make the Notify menu node depend on it. The configuration group keeps only the ubus file list method and the httping, nping and arping list entries it needs for the tracking-binary presence check. A configuration delegate now gets no access to the hook by default. Read on the new group lets them view it read-only, and write lets them edit it. The Notify page also hides its Save button when the session lacks write.

      +
      +

      luci-app-mwan3: escape config-derived values in the status views

      +

      LuCI's E() and dom.append assign a bare string child to a node's innerHTML, where it is parsed as HTML, but append an array child or a node as escaped text. The mwan3 status and analysis views passed UCI-derived free-form values straight in as bare strings. A user who can write mwan3 UCI could store markup in any of these that then runs in the browser of anyone who later opens the status pages.

      +

      Add a small helper that returns a text node, change the shared inline primitives to append their text as an array child so it is never written to innerHTML, and pass each of these values through the helper. The troubleshooting view's diagnostic output is likewise appended as an array child, which both escapes it and keeps it verbatim.

      +
      +

      luci-app-mwan3: refactor and modularise

      +

      App-wide refactor to use the published, supported client-side API throughout and modularisation to support maintainability.

      +

      Logic that had accumulated as copies across the individual views is factored into a small shared library that the views compose from: render helpers, value formatters, address and set arithmetic, input validators, the interactive widgets, and the default constants. Checks and defaults that were restated in several places now have a single definition. The result is much less code per view and a smaller footprint for future changes.

      +

      Visible behaviour and the configuration model are unchanged, aside from a few incidental display corrections.

      +
      +

      luci-app-mwan3: update routing health display

      +

      mwan3 now installs the per-interface unreachable rule as a static backstop: it is created for every configured interface when the service starts and removed only when the service stops. It is therefore present regardless of whether the interface is online, offline or disabled, and it survives a hard ifdown. The routing health tab historically treated that rule as one that comes and goes with the interface, so it presented the now-normal presence of the backstop on a downed interface as exceptional.

      +

      Also correct how the tab describes a soft tracker failure: only a hard ifdown removes the iif and fwmark rules and flushes the table.

      +

      Update the display to reflect the actual rule lifecycle:

      +
        +
      • Presence of rules on an offline interface is expected.
        • +
      • Show a disconnecting event as yellow-bordered.
        • +
      • Update the field guide: a soft tracker failure retains all of the interface's rules; a hard ifdown removes the iif and fwmark rules and flushes the routing table, leaving intact the unreachable rule.
        • +
      +
      +

      18.4 Version 3.6.9

      Summary: Stop router-originated IPv6 link-local traffic, most visibly odhcpd's DHCPv6 replies to LAN clients, from being policy-routed into a WAN routing table and failing, by accepting single-link scopes before classification. Clear an inherited skb mark on ingress so a tunnelled-WAN reply is no longer pinned to the wrong routing table. Emit dnsmasq nftset= directives correctly when a domain is shared across multiple ipset sections, and stage the fragment atomically under the runtime directory.

      mwan3: clear inherited mark on ingress before conntrack restore

      @@ -1632,13 +1957,13 @@ mwan3-lb-test cleanup

      Link-local traffic can never leave its link, so policy-routing it is never correct. Accept it before classification in mwan3_output and mwan3_prerouting. Covered are exactly the scopes that are single-link by definition (RFC 4291): link-local unicast (fe80::/10) and interface-/link-scope multicast (ff01::/16, ff02::/16). Wider multicast scopes remain subject to policy routing.

      The existing ICMPv6 ND bypass stays: NS/NA can also target global unicast addresses, which a link-local destination match does not cover. Its stale comment is rewritten.

      -

      18.2 Version 3.6.8

      +

      18.5 Version 3.6.8

      Summary: Fix an operator precedence regression in 3.6.7 in the address family guard.

      mwan3: fix operator precedence in the address-family guard

      The address-family guard in mwan3_set_user_nft_rule()combined its tests as [ ] && [ ] || [ ] && [ ], which POSIX shell evaluates as ((A && B) || C) && D rather than the intended (A && B) || (C && D), so the guard no longer skipped a mismatched-family pass. Group each && pair to restore the intended behaviour.

      -

      18.3 Version 3.6.7

      +

      18.6 Version 3.6.7

      Summary:

      Comma-separated address lists are now accepted in src_ip and dest_ip rule options, with each element individually validated and the list converted to nft anonymous set syntax at rule-build time.

      IP address validation has been overhauled. A new mwan3ipcheck binary replaces the fragile regex-based validation, using the inet_pton() function for address parsing that correctly rejects invalid input the old regex accepted silently. Of a list of 168 test cases, including both valid and invalid addresses, the regex expressions incorrectly passed 53 that were syntactically incorrect, meaning that mwan3ipcheck is a substantial reliability enhancement.

      @@ -1668,7 +1993,7 @@ mwan3-lb-test cleanup

      During recovery (score <= up), mwan3track logged an info message for every track IP that responded successfully, every tracking round. With multiple track IPs and a default up threshold of 5, this produced up to 15 info messages per recovery episode.

      The connecting and connected notices already communicate recovery progress at the operational level. Per-host probe results during recovery are diagnostic detail, now available via the verbose_logging option.

      -

      mwan3: add verbose_logging option to enable debug log output

      +

      mwan3: add verbose_logging option to enable debug log output

      The LOG() function has historically suppressed all debug-level messages unconditionally, making the existing debug log calls across common.sh, mwan3.sh, and mwan3track permanently dead code.

      Add a globals UCI option verbose_logging (default 0) that controls whether debug-level messages are emitted to syslog. When enabled, all existing LOG debug calls become active, providing detailed diagnostic output for troubleshooting interface tracking and nft rule setup.

      The option is loaded in mwan3_init() for all scripts, and also in mwan3track's load_tracking_config() so that changes take effect on HUP-triggered config reloads without requiring a full restart.

      @@ -1685,7 +2010,7 @@ mwan3-lb-test cleanup

      When a signal (HUP, USR1, or USR2) arrives while mwan3track is mid-way through a ping round, the signal handler kills the current ping subprocess and sets an event flag. The killed ping returns non-zero, and the for loop continues to remaining tracking targets, all of which also fail. The event flags are not checked until after the for loop completes, so the entire round of killed-ping failures is scored and logged as genuine tracking failures.

      Add an event flag check inside the per-IP for loop that breaks immediately when any signal flag is set. After the for loop, if any flag is set, reset the round counters and continue to the top of the while loop where event handling already runs, skipping the scoring, logging, and sleep for the aborted round.

      -

      mwan3: support comma-separated addresses in src_ip and dest_ip rule options

      +

      mwan3: support comma-separated addresses in src_ip and dest_ip rule options

      The mwan3ipcheck binary returns "mixed" for comma-separated address lists containing both IPv4 and IPv6 elements. Add "mixed" to the validation rejection check alongside "invalid", since a single nft anonymous set cannot contain both address families.

      When a comma is present in src_ip or dest_ip, convert the value to nft anonymous set syntax by wrapping in { } with normalized spacing. Single addresses (no comma) pass through unchanged, preserving identical nft output to prior behaviour.

      CIDR elements within comma lists are supported natively by nft anonymous sets and require no special handling.

      @@ -1720,7 +2045,7 @@ mwan3-lb-test cleanup

      Work around a LuCI framework bug where the datatype validator corrupts the value passed to custom validate functions for CIDR inputs. The ipset entry validator now performs its own format checking via a stub validator instead of relying on o.datatype.

      Improve the configuration checker's rule shadowing detection to recognise when two rules reference the same nftset, rather than conservatively skipping all nftset comparisons.

      -

      18.4 Version 3.6.6

      +

      18.7 Version 3.6.6

      Summary: A race condition in the ct mark save logic has been fixed. The previous two-step clear and set sequence first cleared the MMX bits to zero and then set the new value in a consecutive rule, but between those two rules, under the right conditions, a packet on another CPU core could read the intermediate zero state, causing connections to be re-evaluated by the rules chain instead of being pinned to their assigned WAN. The clear is now folded into each setter chain as a single atomic expression, eliminating the race window and ensuring clear-set atomicity.

      Adds a startup guard: mwan3 exits cleanly during startup if all interfaces are disabled, avoiding unnecessary nft chain loading and daemon startup. The default configuration now disables all interfaces and rules. Previously, a fresh install with no explicit user configuration would break the IPv6 internet because the balanced policy had no enabled IPv6 members, causing all IPv6 traffic to hit fall through to the unreachable last resort. A fresh install is now inert until explicitly configured by the user.

      @@ -1737,14 +2062,14 @@ mwan3-lb-test cleanup

      Fold the clear into each save setter chain so that ct mark goes directly from old-value to new-value in a single nft expression (ct mark set ct mark & COMPLEMENT | VALUE). This eliminates the intermediate zero state entirely, providing the same single-write atomicity that iptables CONNMARK --save-mark had.

      The standalone clear rules in the prerouting and output chains are removed since the setter chains now handle the masking internally.

      -

      18.5 Version 3.6.5

      +

      18.8 Version 3.6.5

      Summary: Replaces all ip command output parsing with direct ucode-mod-rtnl netlink calls via four new helper scripts in /lib/mwan3: mwan3-create-iface-route.uc, mwan3-get-addr.uc, mwan3-list-routes.uc, mwan3-manage-rules.uc. These scripts replace the shell invocations of the ip binary and consequent (fragile) parsing of the ip output using sed, awk and grep, helping to make mwan3 more robust and future proofing it against potential changes in the output format of ip commands, since the helper scripts return precisely the information needed by mwan3 and do not require any parsing of the output. The scripts also contribute to a substantially enhanced efficiency profile, since there is now only one call per operation instead of multiple ip subprocesses each invoking ip, sed, grep and awk.

      Adds mwan3ct, a small C-language replacement for the conntrack user-space CLI tool that takes over all conntrack flush operations previously handled by the conntrack CLI tool. mwan3ct makes possible several highly targeted flush operations currently not possible with the conntrack CLI tool without extensive (fragile) parsing of the output and multiple invocations of conntrack -D.

      mwan3ct flushes UNREPLIED conntrack entries when an interface comes online to prevent stale marks from pinning traffic to the wrong WAN. This was previously not possible and is now enabled via mwan3ct. It's particularly helpful in the scenario of a load-balanced connection that has keepalives that could pin a connection to a failed WAN because the keepalives maintain the conntrack entry current.

      Applies minor hardening by moving nft batch files from /tmp to the root-owned /var/run/mwan3 directory and applies more restrictive umasks to the directory and file creation.

      Fixes a scoping issue / buggy behaviour in nft that returns ipsets from other tables even though the query is scoped to a specific table, leading to a slow load time on luci-app-mwan3 Ipset and Rule tabs if the other tables contain large set-based blocklists (eg., adblock packages, banIP).

      -

      mwan3: fix nftset_info fetching all inet family sets

      +

      mwan3: fix nftset_info fetching all inet family sets

      nft 1.1.6 does not scope nft list sets <family> <table> to the named table - it returns every set in the family, including sets from unrelated tables. On systems with firewall packages that maintain large element sets (e.g. IP blocklists), nft -j list sets inet mwan3 would return several megabytes of JSON containing elements from those foreign sets, making the rpcd nftset_info call take much longer than it should.

      Replace nft -j list sets inet mwan3 with nft -j list table inet mwan3, which is correctly scoped to the mwan3 table and returns only its content.

      Since nft list table includes set element data inline, the separate count_nftset_elements() subprocess call per set is also eliminated: replace it with a direct length() of the elem array already present in the parsed JSON.

      @@ -1815,14 +2140,14 @@ mwan3-lb-test cleanup

      The is_public_v4() function in mwan3-diag only excluded loopback, RFC1918, and link-local addresses. The "this network" block (0.0.0.0/8, RFC 1122) and multicast/reserved ranges (224.0.0.0/4 and above) were not excluded, causing them to be replaced with placeholder addresses in the diagnostic output rather than shown verbatim.

      Add exclusions for 0.0.0.0/8 and 224.0.0.0/3 so that these non-routable addresses are preserved in diagnostic output alongside the other non-routable ranges already excluded.

      -

      18.6 Version 3.6.4

      +

      18.9 Version 3.6.4

      Summary: Fixes a regression in 3.6.3 that prevents mwan3's hotplug handler from running when the service is started but not explicitly enabled

      mwan3: remove enabled check from hotplug handler

      The enabled guard added in b363e9b50 causes the hotplug handler to exit silently when the rc.d symlink is absent. This breaks the startup path: start_service spawns per-interface hotplug handlers via mwan3_ifup using env -i, and those handlers exit at the enabled check without creating per-interface nft chains, ip rules, or routing tables. All WAN-bound traffic then hits the unreachable last-resort rule and connectivity is lost.

      The guard was intended to prevent hotplug side effects when mwan3 is disabled, but it is redundant. The nft table existence check that follows already covers every practical scenario: the table is created by start_service and deleted by stop_service, so it is present only while the service is running. A service that was started manually without being enabled should still process hotplug events.

      -

      18.7 Version 3.6.3

      +

      18.10 Version 3.6.3

      Summary: Correctness fixes for edge cases and incomplete feature implementations: fixes ipset and ICMP protocol translation issues when rules use family=any; rebuilds ip rules and routing tables on reload to handle interface reordering and family changes; completes track_gateway support that was missing from two tracking status checks; adds runtime configuration reload to mwan3track via a SIGHUP handler and fixes hotplug handler initialization issues and boot race conditions.

      mwan3: fix ipset+family=any batch failure and sticky chain orphan

      @@ -1844,7 +2169,7 @@ mwan3-lb-test cleanup

      A secondary manifestation of the same root cause: if an interface's family option changes (e.g. ipv4 to ipv6), the old ip rules remain in the wrong address family and new rules are never created.

      Add mwan3_rebuild_iface_rules, called from reload_service via config_foreach, which deletes and recreates ip rules and routing tables for each enabled, up interface. Modify mwan3_delete_iface_rules and mwan3_delete_iface_route to search both address families unconditionally, so that a family change cleans up rules left behind in the old family.

      -

      mwan3: fix mwan3rtmon has_tracking ignoring track_gateway

      +

      mwan3: fix mwan3rtmon has_tracking ignoring track_gateway

      The load_config function in mwan3rtmon determines whether an interface has tracking enabled by checking only track_ip. Interfaces configured with track_gateway but no track_ip are incorrectly treated as having tracking disabled. This causes get_track_status to return "disabled" without consulting the tracker's PID and STARTED state files, bypassing the secondary health check guard in handle_route_event.

      The shell equivalent in common.sh (mwan3_get_mwan3track_status) and the init script (start_tracker) both correctly account for track_gateway. Align mwan3rtmon's has_tracking logic to match by including track_gateway in the check.

      @@ -1853,7 +2178,7 @@ mwan3-lb-test cleanup

      Add a load_tracking_config() function that extracts the parameter reads from main() into a reusable function called both at startup and on reload. Add a HUP signal handler to mwan3track that sets a reload flag and interrupts the current sleep/probe cycle. The main loop checks the flag at the same two points where IFDOWN and IFUP events are processed, re-reads UCI config, refreshes tracking parameters and track IPs, and clamps the score to the new down+up ceiling if it exceeds it.

      In reload_service, send HUP to each running tracker instance after the existing rtmon HUP signals.

      -

      mwan3: fix mwan3_get_mwan3track_status ignoring track_gateway

      +

      mwan3: fix mwan3_get_mwan3track_status ignoring track_gateway

      The mwan3_get_mwan3track_status function in common.sh returns "disabled" when no track_ip list entries are configured, without checking the track_gateway option. This causes the "mwan3 interfaces" CLI command to report "tracking is disabled" for interfaces that use gateway-only tracking, even though a tracker instance is actively running and monitoring the interface.

      The rpcd ucode equivalent (get_mwan3track_status) already checks both track_ip and track_gateway before returning "disabled". Align the shell function with the same logic by also reading track_gateway before the early return.

      @@ -1862,13 +2187,13 @@ mwan3-lb-test cleanup

      Move the nft table existence check to before mwan3_init. Previously mwan3_init was called unconditionally, which created /var/run/mwan3 as a side effect and corrupted the stopped-service indicator in the case where mwan3 is enabled but has been manually stopped. With the check before mwan3_init, the handler exits without side effects whenever the nft table is absent, covering both the boot-race window and the stopped-but-enabled case.

      Remove the running check that followed mwan3_init. It was dead code: mwan3_init created /var/run/mwan3 before the check ran, so service_running() always returned true and the exit path was unreachable.

      -

      18.8 Version 3.6.2

      +

      18.11 Version 3.6.2

      Summary: A set of defensive edge-case fixes and correctness improvements. Binds the mwan3rtmon route listener before the initial netlink dumps to close a narrow startup race window, and replaces main_route_cache with an on-demand kernel query to eliminate a class of cache-drift failures that could only manifest if route events arrived during the dump phase. Aligns shell and mwan3rtmon custom-set filtering so both paths apply identical exclusions. Adds a dormant is_default_route guard to populate_connected_set as a forward-compatibility precaution. Clamps check_quality to 0 when the configured track method cannot produce quality samples, preventing an arithmetic error in the unusual case where check_quality is paired with a non-ping method. Fixes mwan3_track_clean which targeted incorrect paths and was a no-op, and tightens ip rule deletion at stop_service to use content-based matching rather than a priority-range regex, which matters only when rule bases are configured outside the default 1000-3999 band.

      mwan3: exclude default routes from mwan3rtmon connected set population

      populate_connected_set filters routes with is_cidr_route and is_linklocal_route but does not check is_default_route. Currently dormant because rtnl returns dst=null for default routes, which is_cidr_route already rejects. Add an explicit is_default_route guard so the filter remains correct if the rtnl module ever emits "0.0.0.0/0" or "::/0" as a string instead of null.

      -

      mwan3: replace main_route_cache with on-demand kernel query in mwan3rtmon

      +

      mwan3: replace main_route_cache with on-demand kernel query in mwan3rtmon

      main_route_cache replaced an on-demand kernel query (route_still_exists, originally from e119a57e9) with a route_key map built from the initial dump and maintained incrementally from listener events.

      The cache made the per-event ECMP-suppress check O(1) but introduced a class of dump-vs-listener race failure modes: any event arriving between the snapshot used to build the cache and the listener becoming active can double-count or undercount the cache, leading to stuck suppressions or premature cleanups of per-interface routing table entries with no self-healing path. The listener-first swap in an earlier commit shifts but does not eliminate this race.

      route_still_exists queries the kernel directly on every delete and is unaffected by cache drift. Cost is one RTM_GETROUTE dump per DELROUTE event whose per-iface table entry exists, single-digit milliseconds on typical routing-table sizes and infrequent in practice.

      @@ -1884,13 +2209,13 @@ mwan3-lb-test cleanup

      Also add an explicit repopulate_custom_sets() call at startup so mwan3rtmon reconciles the mwan3_custom_v4/v6 sets with kernel state once it begins running. The shell seeds these sets during start_service, but custom-table route changes between the shell pass and mwan3rtmon startup are unobserved by either side; the startup dump closes that window.

      The connected set is unchanged; it already self-heals through a debounced rebuild triggered by every CIDR-route event.

      -

      mwan3: clamp check_quality for track methods that lack loss/latency

      +

      mwan3: clamp check_quality for track methods that lack loss/latency

      mwan3track's per-iteration case statement only populates $loss and $latency for the ping and httping methods. arping, nslookup, and nping-* set only $result. With check_quality=1 configured against one of those methods the post-case decision block reaches

      [ "$loss" -ge "$failure_loss" ] || [ "$latency" -ge ... ]

      with empty (or stale from a previous iteration) operands, which raises a busybox ash arithmetic error and leaves the per-host up/down classification undefined.

      Clamp check_quality to 0 at startup when the configured track_method cannot produce quality samples, and emit a notice so the user can see that the option was overridden. Apply the same clamp in mwan3_load_track_ips so that LATENCY_/LOSS_ state files are still cleaned up after a config edit that transitions the interface from a quality-capable method to one that is not.

      -

      mwan3: clean correct per-interface state paths in mwan3_track_clean

      +

      mwan3: clean correct per-interface state paths in mwan3_track_clean

      mwan3_track_clean is called from stop_service via mwan3_interface_shutdown to tear down per-interface runtime state. The original implementation removed $MWAN3_STATUS_DIR/<iface> and attempted to rmdir $MWAN3_STATUS_DIR. Neither matched where the per-interface state actually lives:

      • tracker runtime state at $MWAN3TRACK_STATUS_DIR/<iface>/(PID, STATUS, LATENCY_*, LOSS_*, TRACK_*, GATEWAY, ...), written by mwan3track;
        • @@ -1900,7 +2225,7 @@ mwan3-lb-test cleanup

        Point the rm at the correct per-interface paths and replace the bogus rmdir with a best-effort rmdir of $MWAN3TRACK_STATUS_DIR itself, which is the dir that should be empty once every iface has been cleaned. $MWAN3_STATUS_DIR is left intact because it holds state that must survive a stop/start cycle.

        Errors are redirected to /dev/null so transient mismatches do not hit the console or logs. A comment in the function documents a small residual race with mwan3track that can briefly recreate the tracker dir between rm and procd_kill; the leak is bounded to one iteration and harmless on tmpfs.

        -

        mwan3: identify ip rules to delete precisely at stop_service

        +

        mwan3: identify ip rules to delete precisely at stop_service

        stop_service identified mwan3-owned ip rules with a fixed regex match on ^[1-3][0-9]{3}:, limiting cleanup to priorities 1000-3999 and deleting every rule in that band regardless of origin. Two problems:

        1. When iif_rule_base, fwmark_rule_base or unreachable_rule_base are configured outside 1000-3999 (all three are exposed as globals and validated in mwan3_init), the affected tier of rules is leaked across service mwan3 stop and accumulates across stop/start cycles.
          2. @@ -1917,7 +2242,7 @@ mwan3-lb-test cleanup

      To make the gates trustworthy at stop time, mwan3_init now persists iif_rule_base, fwmark_rule_base and unreachable_rule_base to state via uci_toggle_state at start_service, mirroring the existing iface_max persistence, and reads them back via uci_get_state on subsequent invocations. The state values reflect the rule bases the running instance actually created its rules with, even if /etc/config/mwan3 has been edited since start. The mmx_mask state file doubles as the "instance started" indicator. The validation step that reverts to default bases on ordering-constraint violation runs only in the fresh-start branch, so persisted values are always post-validation.

      -

      18.9 Version 3.6.1

      +

      18.12 Version 3.6.1

      Summary: Extends the legacy mwan3 custom sets that were previously only loaded statically during start_service() and reload_service() from the tables defined in the UCI global config list option rt_table_lookup to be fully dynamic, using mwan3rtmon to listen for and to add and remove routes from the custom sets in response to RTM_NEWROUTE and RTM_DELROUTE events on the tables defined with list rt_table_lookup <tableid>. Adds a SIGHUP handler to mwan3rtmon to cause it to flush and repopulate these custom sets, ensuring that their contents remain in sync with any newly added or removed list rt_table_lookup <tableid> options in the mwan3 config.

      mwan3: reload mwan3rtmon config on mwan3 reload via SIGHUP

      @@ -1926,19 +2251,19 @@ mwan3-lb-test cleanup

      reload_service() in the init script sends SIGHUP to the rtmon_ipv4 (and rtmon_ipv6 if IPv6 is enabled) procd instances via procd_send_signal after mwan3_nft_reload_commit and the ip rule updates, so the nft sets and ip rules are fully consistent before mwan3rtmon re-dumps.

      The shell-level mwan3_set_custom_sets() call already present in reload_service() is retained. It runs inside the atomic nft batch and provides a synchronous static snapshot that ensures the sets are never empty during the reload window. The SIGHUP-triggered re-dump that follows is the authoritative update because it runs after the batch commits and picks up any route changes that occurred in the interval between the shell dump and the signal delivery.

      -

      mwan3: handle rt_table_lookup route events dynamically

      +

      mwan3: handle rt_table_lookup route events dynamically

      The rt_table_lookup feature populated the mwan3_custom_v4 and mwan3_custom_v6 nftables sets at startup by reading existing routes from the configured tables, but did not update those sets when routes changed at runtime. Route additions and deletions in rt_table_lookup tables were therefore not reflected until mwan3 was restarted.

      Add handle_custom_set_event(), called from handle_route_event() whenever a route event arrives for a table listed in extra_table_set. RTM_NEWROUTE events add the destination to the appropriate custom set; RTM_DELROUTE events remove it. Default routes and link-local routes are excluded, matching the exclusions already applied at startup.

      -

      18.10 Version 3.6

      +

      18.13 Version 3.6

      Summary: Version 3.6 adds three user-visible features to mwan3 rules. Rules now support an fwmark/fwmask option to match packets by meta mark using a masked comparison, working alongside or instead of address and ipset matching; mwan3 logs a warning if the fwmask overlaps its internal MMX_MASK since such a mask would match packets already carrying an mwan3 classification mark. The ip rule priority tiers for per-interface rules are now configurable via three new globals UCI options (iif_rule_base, fwmark_rule_base, unreachable_rule_base), shifting from the fixed 1000/2000/3000 defaults; two ordering constraints are enforced at startup and rule deletion is rewritten to use content-based matching so it remains correct across base or mmx_mask changes. Rules gain option enabled 0/1, consistent with interfaces, ipsets, and members.

      The LuCI interface is updated throughout to reflect all three additions. The rule modal gains an Fwmark field with MMX_MASK overlap validation; the Globals tab gains the three base priority fields with live cross-field ordering validation and an automatic mwan3 restart when any base changes. The Routing Health tab gains the unreachable rule row per interface and marks an interface as degraded if the unreachable rule is absent; base priorities are now displayed dynamically from the rpcd endpoint rather than assumed from fixed offsets. The Traffic Simulator, rule shadowing analysis, Status Overview, and diagnostics helper are all updated to handle fwmark matching, disabled rules, and configurable bases. Mutual enable protection is added to prevent the silent misconfiguration of an enabled rule referencing a disabled IP set. mwan3track now validates that libwrap_mwan3_sockopt.so is present at startup, exiting with a clear error rather than silently producing incorrect tracking results if the library is missing.

      -

      mwan3: add per-interface unreachable rule to rpcd routing_health

      +

      mwan3: add per-interface unreachable rule to rpcd routing_health

      The routing_health() rpcd endpoint reported iif and fwmark rule presence per interface but omitted the unreachable rule. This meant the Routing Health tab could not detect a missing unreachable rule, and would show a fully healthy interface even if its unreachable safety net was absent.

      Add unreach_rule (present + priority) to each interface in the response, and include the unreachable base priority in the rule_bases object returned to the frontend.

      -

      mwan3: update rpcd routing_health for configurable bases and dynamic mmx_mask

      +

      mwan3: update rpcd routing_health for configurable bases and dynamic mmx_mask

      The routing_health() function in the rpcd ucode module used hardcoded constants for the ip rule base priorities (1000/2000) and the maximum interface count (63). This meant the Routing Health tab reported incorrect data when configurable rule base priorities were in use, and would also be wrong for any installation with a non-default mmx_mask.

      Read iif_rule_base, fwmark_rule_base, and unreachable_rule_base from UCI globals with the same defaults and ordering constraint validation as mwan3_init in common.sh. Derive mmdefault and iface_max dynamically from the configured mmx_mask by counting set bits, replacing the hardcoded MAX_IFACES constant. This ensures the blackhole and unreachable global policy rule priorities, stale rule range detection, and ordering constraint validation all reflect the actual runtime configuration.

      Add the unreachable rule tier to valid priority tracking and stale rule detection. Previously only the iif and fwmark tiers were checked, so per-interface unreachable rules were invisible to the health report and could be falsely flagged as stale.

      @@ -1958,7 +2283,7 @@ mwan3-lb-test cleanup

      mwan3-diag imports the ucode socket module to normalise IPv4 and IPv6 address strings via sock.sockaddr(). The ucode-mod-socket package was not listed in DEPENDS, so on a fresh install the module would be absent and mwan3-diag would abort at startup with an import error.

      Add +ucode-mod-socket to the package DEPENDS.

      -

      mwan3: wire validate_wrap() startup check for libwrap_mwan3_sockopt

      +

      mwan3: wire validate_wrap() startup check for libwrap_mwan3_sockopt

      If libwrap_mwan3_sockopt.so is absent, mwan3track now exits with a clear error at startup rather than silently producing incorrect tracking results. Also define the library path as a single WRAP_LIB constant, and fix the file test from -x to -f.

      mwan3: add option enabled support for rules

      @@ -2015,30 +2340,30 @@ mwan3-lb-test cleanup

      To accommodate the new column without widening the grid, the separate Source port and Destination port columns are merged into the Source and Destination columns respectively. The merged textvalue format is ddress:port or ipset_name:port when both are set, address or ipset_name when only an address is present, *:port when only a port is set, and - when neither is set. src_port and dest_port are marked modalonly so they remain editable via the modal.

      The Policy assigned column label is shortened to Policy.

      -

      18.11 Version 3.5.3

      +

      18.14 Version 3.5.3

      Summary: Version 3.5.3 adds two major LuCI features and a set of bug fixes and routing reliability improvements.

      The Policy tab is rewritten with a tier-based policy builder that lets users specify failover tiers and per-interface load-balancing weights without manually creating member definitions. The builder handles member creation and garbage-collects orphaned members automatically, and updates the policy list to display IPv4 and IPv6 policies in symbolic form. The traffic simulator gains hostname resolution: Source and Destination fields now accept hostnames as well as IP addresses, resolved via the new resolve_host rpcd method against the local DNS server. The new Policy tab metaphor and layout should greatly improve how both new and existing users get to grips with mwan3, whose flexible but less than intuitive interface-member-policy definitions sometimes pose a substantial barrier to immediate use.

      Three new rpcd methods - nftset_flush, nftset_reload, and nftset_resolve - allow the LuCI IP Sets tab to flush a set, repopulate it from UCI static entries and a loadfile, or trigger fresh DNS resolution by sending SIGHUP to dnsmasq before resolving configured domains. Matching Flush, Reload, and Resolve buttons appear on each set panel alongside the existing Expand button. get_nftset_members is fixed to unwrap the counter-decorated element wrapper that nft emits for counter-enabled sets, which previously caused all members of such sets to be silently dropped, breaking simulator rule matching for counter-enabled nftsets.

      mwan3_create_iface_route is changed to use ip route replace instead of ip route add, making route installation unconditionally idempotent and eliminating spurious EEXIST errors when mwan3rtmon or a racing event had already inserted the route. A missing default argument in the config_get_bool call for the enabled option is added, suppressing cosmetic sh: out of range noise on startup.

      -

      mwan3: add nftset_flush, nftset_reload, and nftset_resolve RPC methods

      +

      mwan3: add nftset_flush, nftset_reload, and nftset_resolve RPC methods

      nftset_flush empties a named nft set via nft flush set.

      nftset_reload flushes the set then repopulates it from the UCI ipset config: static entries from the entry list and any entries read from the loadfile path.

      nftset_resolve sends SIGHUP to dnsmasq to clear its cache, then iterates the domain list for the ipset and calls nslookup_resolve for each, triggering fresh DNS lookups that cause dnsmasq to populate the set via its nftset integration.

      -

      mwan3: fix missing default in config_get_bool enabled call

      +

      mwan3: fix missing default in config_get_bool enabled call

      mwan3_update_dev_to_table iterates all mwan3 interface sections to build the device-to-routing-table map. The config_get_bool call for the enabled option had no default argument. When an interface section has no explicit option enabled set, get_bool receives an empty value and an empty default, falls through to the wildcard case, and returns an empty string. The subsequent [ "$enabled" -eq 0 ] comparison then receives an empty string as its left operand, causing busybox ash to emit sh: out of range (empty string has no digits consumed, triggering the error path in the test builtin's integer conversion).`

      The error is cosmetic. The failed test exits non-zero, so the && return does not fire and the interface is treated as enabled, which is correct, but the error messages are misleading and concern users.

      Fix by adding the missing default of 1, consistent with all other config_get_bool enabled calls in the file.

      -

      mwan3: use ip route replace in mwan3_create_iface_route

      +

      mwan3: use ip route replace in mwan3_create_iface_route

      The previous ip route add would fail with EEXIST if the route was already present in the per-interface table -- for example when mwan3rtmon had already copied it from the main table, or when a connected event raced an ifup event. The string-match dedup check intended to prevent this was fragile: ip route list output for the same route can differ in text representation between table main and a per-interface table (field ordering, explicit metric 0, etc.), causing the check to miss a match and fall through to a duplicate add.

      ip route replace uses NLM_F_CREATE | NLM_F_REPLACE at the kernel level, making the operation unconditionally idempotent. This is the same semantics mwan3rtmon already uses for its route copies. The duplicate-detection pre-check is retained as a cheap optimisation to avoid a redundant syscall when the route is already known to be present, but it is no longer safety-critical.

      -

      mwan3: add resolve_host RPC method for hostname resolution

      +

      mwan3: add resolve_host RPC method for hostname resolution

      Add nslookup_resolve() helper and resolve_host RPC method to allow the LuCI traffic simulator to accept hostnames in addition to IP addresses. The resolver invokes /bin/busybox nslookup against the local DNS server (127.0.0.1), populating any dnsmasq nftset entries configured for the queried domain as a side effect. Returns separate v4 and v6 address arrays; the family parameter restricts resolution to A or AAAA records. Input is validated against [a-zA-Z0-9._-]+ before use.

      -

      mwan3: fix get_nftset_members to unwrap counter-decorated elements

      +

      mwan3: fix get_nftset_members to unwrap counter-decorated elements

      nft wraps set elements in {"elem":{"val":"...","counter":{...}}} objects when the set has the counter flag. The previous code passed these wrapper objects directly to parse_elem_val(), which returned null, silently dropping all members of counter-enabled sets. This caused the traffic simulator to fail to match rules that reference counter-enabled nftsets.

      get_nftset_elements() already handled this case correctly; apply the same unwrap logic to get_nftset_members().

      @@ -2060,7 +2385,7 @@ mwan3-lb-test cleanup

      When a hostname is entered, the simulator calls the resolve_host RPC method to resolve it via the local DNS server before running the simulation. The resolved address is displayed inline as Resolved: x.x.x.x (+N more). Resolution errors abort the simulation with an error message.

      The address family selector controls A vs AAAA record resolution; IPv4 is preferred when the selector is set to both families. Update the ACL to permit the resolve_host method, update field labels and placeholder text to indicate hostname support, and widen the input fields.

      -

      18.12 Version 3.5.2

      +

      18.15 Version 3.5.2

      Summary: Version 3.5.2 is a bug-fix and maintenance release. It corrects a misrouting bug where kernel-generated NDP Neighbor Solicitation probes entered mwan3_output without a conntrack entry, fell through to mwan3_rules, and received a WAN policy mark that caused the kernel to probe the gateway via the wrong interface, cycling the NDP entry to FAILED state and breaking WRAP ping tracking for that interface. It updates the package dependency from ip to ip-full to ensure the full iproute2 implementation is always present, since the busybox ip is a minimal subset that does not support all options mwan3 requires. It adds mwan3-diag, a ucode diagnostic script installed to /usr/sbin/mwan3-diag that collects a comprehensive snapshot of mwan3 state -- interface status, policy routing rules, nftables ruleset, routing tables, conntrack summary and system log -- with all public IP addresses anonymised with stable placeholders so output can be shared safely.

      mwan3: add mwan3-diag network diagnostic script

      @@ -2069,11 +2394,11 @@ mwan3-lb-test cleanup

      mwan3: depend on ip-full instead of ip

      The busybox ip implementation is a minimal subset of iproute2 and does not support all options and subcommands that mwan3 requires for correct operation. Depend on ip-full to ensure the full iproute2 implementation is always present.

      -

      mwan3: bypass NDP in mwan3_output to prevent re-routing of NDP probes

      +

      mwan3: bypass NDP in mwan3_output to prevent re-routing of NDP probes

      Kernel-generated NDP Neighbor Solicitation probes start with mark=0 and enter mwan3_output. They have no conntrack entry, so the ct mark restore is a no-op. They are not matched by mwan3_connected, mwan3_custom, or mwan3_dynamic. They fall through to mwan3_rules, where the default IPv6 rule (ip6 daddr ::/0) applies a WAN policy mark -- the same mark that would be assigned to outbound user traffic. Because mwan3_output is type route, the mark change triggers a routing re-evaluation, which may route the probe to a different WAN interface than the one whose gateway the kernel is trying to resolve. The gateway NDP entry cycles to FAILED state, and subsequent WRAP ping probes are dropped because the kernel cannot resolve the gateway MAC address.

      Add an icmpv6 NDP accept rule at the top of mwan3_output, mirroring the equivalent rule already present in mwan3_prerouting.

      -

      18.13 Version 3.5.1

      +

      18.16 Version 3.5.1

      Summary: Version 3.5.1 is a bug-fix and maintenance release. It corrects a silent failure in mwan3rtmon where route replication to per-interface routing tables was completely non-functional, adds nft set flag-change detection on reload so that changing a set's timeout, counter, or size options takes effect immediately without requiring a full service restart, suppresses spurious stderr noise from ip rule and ip route operations during upgrades and teardown, and removes version number references from comments.

      mwan3: fix mwan3rtmon route replication broken by stale table name

      @@ -2106,18 +2431,18 @@ mwan3-lb-test cleanup
    • mwan3_set_custom_set: ip route list calls restructured to capture output and check exit code separately, so a missing rt_table_lookup table is suppressed on stderr but logged via LOG warn. This preserves the error as a diagnosable signal without printing raw kernel errors to the console.
    -

    18.14 Version 3.5

    +

    18.17 Version 3.5

    Summary: Version 3.5 is a major architectural release that moves mwan3 out of table inet fw4 and into its own table inet mwan3, eliminating the fw4 rebuild scaffold and the mwan3evtd debounce daemon entirely.

    The reload path is replaced with a single atomic nft batch that commits the complete new ruleset while the old one is still serving traffic, with zero window of misrouted connections. User-declared nft sets are now configured directly in /etc/config/mwan3 with inline, file, and dnsmasq-populated modes, and per-element packet and byte counters are optionally available. The APK install lifecycle is hardened to eliminate RTNETLINK errors on both fresh install and upgrade. LuCI gains a full IP Sets configuration tab and a new IP Sets status view with paginated element display, and the overview layout is redesigned with CSS grid cards. Additional bug fix to debounce RTM_NEWROUTE calls in mwan3rtmon.

    -

    mwan3: update mwan3rtmon to debounce RTM_NEWROUTE calls

    +

    mwan3: update mwan3rtmon to debounce RTM_NEWROUTE calls

    RTM_NEWROUTE events for connected routes were fast-pathed directly to nft_exec, bypassing the debounce timer that was only applied to deletes. On IPv6 systems with prefix delegation, the kernel appears to send repeated RTM_NEWROUTE updates for already-present connected routes, causing nft processes to be spawned multiple times per second and producing measurable CPU load.

    Fix: route both RTM_NEWROUTE and RTM_DELROUTE for CIDR routes through the same 100ms debounce timer, calling populate_connected_set() once after the burst settles rather than once per event. Add a fingerprint (sorted, joined element list) to opulate_connected_set() so that calls where the connected set content has not changed skip the nft_batch call entirely. Also add ECMP deduplication (seen map) and link-local filtering to the element build loop in populate_connected_set().

    mwan3: add conntrack as a hard dependency

    mwan3 relies on the conntrack userspace tool in several places: flush_conntrack is called when interfaces go down or policies change to force existing connections to be re-evaluated under the new routing state. Without conntrack installed these operations silently fail, leaving stale connections pinned to a dead or reconfigured WAN interface. Making the dependency explicit ensures conntrack is always present when mwan3 is installed.

    -

    mwan3: add counters support, nftset_elements RPC, and orphaned set cleanup

    +

    mwan3: add counters support, nftset_elements RPC, and orphaned set cleanup

    Add option counters (bool, default 0) to config ipset sections. When set, enables per-element packet and byte count tracking via nft set counter statement.

    Change maxelem default from 65536 to 0 (unlimited), matching fw4 behaviour where sets have no size limit unless explicitly configured.

    Add mwan3_cleanup_orphaned_ipsets, called during reload_service after mwan3_render_config_ipsets. Queries nft for user-defined sets not prefixed mwan3_, compares against configured set names, and deletes any orphans. Prevents stale sets accumulating when a set is removed via LuCI and the config is applied.

    @@ -2218,21 +2543,21 @@ mwan3-lb-test cleanup

    luci-app-mwan3: update port-range hint to use dash separator

    mwan3 now renders port ranges using x-y (nft native format); the colon separator x:y is still accepted in UCI for backwards compatibility but is no longer the recommended input format. Update the src_port and dest_port help text from "1024:2048" to "1024-2048" so new entries match nft syntax directly.

    -

    18.15 Version 3.4.1 (Unreleased)

    +

    18.18 Version 3.4.1 (Unreleased)

    Summary: Builds the per-interface mwan3_iface_in_* chains before mwan3_set_general_nft() activates mwan3_prerouting to avoid a race condition that leads to a wrong interface mark being assigned. Fixes bugs in the nft list chains syntax in stop_service() and a grep expression that was causing a too-broad match and resulting in traffic for interface wan bypassing mwan3 marking.

    -

    mwan3: fix nft list chains syntax error in stop_service

    +

    mwan3: fix nft list chains syntax error in stop_service

    nft list chains only accepts an optional family argument, not a table name. The two chain-enumeration loops in stop_service used nft list chains inet fw4, which is invalid syntax. The fw4 argument caused nft to exit with an error; 2>/dev/null suppressed it, producing no output. Both loops therefore silently iterated over nothing on every service mwan3 stop or service mwan3 restart.

    Consequence: dynamic chains (mwan3_iface_in_*, mwan3_policy_*, mwan3_or_meta_*, mwan3_or_ct_*) were never flushed or deleted on stop. They persisted in table inet fw4 as empty orphan objects. The final hardcoded skeleton-chain flush at the end of stop_service was unaffected and continued to work correctly, so functional impact was limited to residual chain objects after stop.

    Fixed by changing both occurrences to nft list chains inet. The grep "chain mwan3_" filter is sufficient to scope results to mwan3's chains, which only exist in table inet fw4.

    -

    mwan3: fix grep substring match in mwan3_ifaces_in chain wiring

    +

    mwan3: fix grep substring match in mwan3_ifaces_in chain wiring

    mwan3_create_iface_nft used grep -q "jump mwan3_iface_in_$1" as an idempotency check before adding a jump from mwan3_ifaces_in to the per-interface chain. grep -q does substring matching, so for $1=wan the pattern also matches lines containing mwan3_iface_in_wan2, mwan3_iface_in_wan6, etc. If those interfaces had already added their jumps, wan's check returned a false positive and its jump was skipped, leaving wan absent from mwan3_ifaces_in and all wan traffic bypassing mwan3 marking entirely.

    The same bug in mwan3_delete_iface_nft caused the handle lookup for wan's jump to return the handle of a different interface's rule.

    The bug existed before this patch but was masked by the old sequential ordering, where wan (first in UCI order) always checked mwan3_ifaces_in before any other process had added jumps. The preceding commit changed to parallel execution, exposing the race.

    Fixed with grep -qw (whole-word match) in both functions. Only affects configurations where one interface name is a prefix of another (e.g. wan/wan2, eth0/eth0b).

    -

    mwan3: build iface_in chains before activating prerouting

    +

    mwan3: build iface_in chains before activating prerouting

    The per-interface mwan3_iface_in_* chains are now fully populated before mwan3_set_general_nft() activates mwan3_prerouting. Previously the ordering was reversed: prerouting became live first, then each mwan3_ifup background process added rules to mwan3_ifaces_in one by one. During that window, packets arriving on any interface whose chain was not yet wired got no ct mark. For DNAT traffic (fib daddr type local return with no mark save) the reply could hit numgen and acquire a random interface mark, which WireGuard PersistentKeepalive then locked in permanently by refreshing the conntrack entry every 25s.

    Changed in all three rebuild code paths:

      @@ -2242,7 +2567,7 @@ mwan3-lb-test cleanup

    Also removed the flush of mwan3_postrouting from mwan3_set_general_nft. That flush was a leftover from when general_nft ran before iface setup; with the new ordering it ran after mwan3_create_iface_nft had already written snat6 rules to postrouting, silently deleting them. mwan3_delete_iface_nft and the per-chain cleanup in mwan3_create_iface_nft already handle postrouting cleanup correctly.

    -

    18.16 Version 3.4

    +

    18.19 Version 3.4

    Summary: Version 3.4 introduces mwan3evtd, a generalised ucode debounce daemon that coalesces rapid-fire events - such as simultaneous interface flaps triggering multiple fw4 reloads - into a single handler execution after the activity settles. This prevents the repeated dnsmasq SIGHUPs that previously caused cache thrash and, in tight-timing scenarios, dnsmasq crashes during concurrent startup.

    mwan3: mwan3evtd debounce daemon

    @@ -2262,14 +2587,14 @@ mwan3-lb-test cleanup

    Shell injection in the handler fire path is prevented by passing the command through EVTD_CMD env var and using eval "$EVTD_CMD", avoiding any brace-grouping that an adversarial } in command could escape.

    mwan3evtd is a generalised debounce daemon, capable of accepting any event type and any handler and can be used by other packages if desired.

    -

    18.17 Version 3.3.5

    +

    18.20 Version 3.3.5

    Summary: Version 3.3.5 is a single-fix release that suppresses the per-deleted-entry output that conntrack -D writes to stdout, which was previously appearing on the console whenever mwan3 start or an fw4 reload triggered the zero-mark conntrack flush.

    mwan3: suppress conntrack -D output to console

    conntrack -D prints each deleted entry to stdout. The call in mwan3_flush_stale_conntrack() redirected only stderr, causing all deleted zero-mark conntrack entries to appear on the console whenever mwan3 start or fw4 reload triggered the flush.

    Fix: redirect stdout to /dev/null alongside stderr.

    -

    18.18 Version 3.3.4

    +

    18.21 Version 3.3.4

    Summary: Version 3.3.4 closes a class of misrouting bugs caused by the brief window between fw4 flushing table inet fw4 and mwan3 completing its nft rebuild. Connections established during that window acquire ct mark=0; the new mwan3_flush_stale_conntrack helper removes all zero-mark conntrack entries after every rebuild and restart, preventing WireGuard persistent-keepalive and similar long-lived UDP from locking in a bad entry indefinitely. A double-rebuild race in mwan3-fw-rebuild.sh is also fixed by acquiring the procd lock before checking for empty chains.

    mwan3: flush zero-mark conntrack entries after fw4 rebuild and restart

    @@ -2288,31 +2613,31 @@ mwan3-lb-test cleanup

    mwan3-fw-rebuild.sh checked for empty mwan3_prerouting before acquiring procd_lock. If 25-mwan3 was already holding the lock and rebuilding, the fw-rebuild script could pass the check, then queue behind 25-mwan3, and proceed to rebuild and call mwan3_dnsmasq_hup` a second time after the lock was released.

    Fix by acquiring procd_lock first and re-checking under the lock, so only one of the two rebuild paths does the actual work.

    -

    18.19 Version 3.3.3

    +

    18.22 Version 3.3.3

    Summary: Version 3.3.3 is a broad bug-fix release addressing several correctness issues: DNAT reply routing was broken by a misplaced fib daddr type local return rule that fired before DNAT translation, causing replies to exit via a randomly load-balanced interface; IPv6 ip rules were silently leaked on ifdown because delete_iface_rules queried the IPv4 rule table; mwan3_dnsmasq_hup never sent SIGHUP because json_get_var stores booleans as integers not strings; and the numgen counter was contaminated by inbound and reply traffic. Additional fixes cover a grep substring false-positive in iface chain wiring, unquoted regex variables, a dead function stub, a duplicate function, and missing mwan3_postrouting in the stop_service chain lists. A new bypass_network UCI option populates the dynamic bypass sets from config, and mwan3-lb-test gains fw4 reload detection.

    -

    mwan3: add mwan3_postrouting to stop_service skeleton chain lists

    +

    mwan3: add mwan3_postrouting to stop_service skeleton chain lists

    mwan3_postrouting is defined in the static 10-mwan3.nft skeleton alongside the other named skeleton chains, but was absent from both the deletion exclusion list and the safety-flush list in stop_service(). This caused it to be deleted on stop rather than preserved, relying on start_service/mwan3_ensure_nft_framework to recreate it. Add it to both lists for consistency with the other skeleton chains.

    -

    mwan3: remove duplicate mwan3_count_one_bits from mwan3.sh

    +

    mwan3: remove duplicate mwan3_count_one_bits from mwan3.sh

    An identical copy of mwan3_count_one_bits() existed in both mwan3.sh and common.sh. Since mwan3.sh sources common.sh, the mwan3.sh copy shadowed the canonical definition without any functional difference. Remove the duplicate; all call sites continue to use the common.sh definition.

    -

    mwan3: remove dead mwan3_set_sticky_nft function

    +

    mwan3: remove dead mwan3_set_sticky_nft function

    mwan3_set_sticky_nft() was an incomplete stub from an earlier sticky routing design that was superseded by the current per-member ip-only set + OR-immediate vmap-dispatch implementation in mwan3_set_user_nft_rule(). The stub had a no-op loop body and was never called. Remove it.

    mwan3_get_policy_members_for_family() is retained - it is called by the active sticky implementation at line 1131.

    -

    mwan3: quote $cmdline in mwan3_get_mwan3track_status

    +

    mwan3: quote $cmdline in mwan3_get_mwan3track_status

    $cmdline was unquoted in the [ $cmdline != ... ] test. If readfile returns nothing (narrow race where the tracked process exits between the PID file read and the /proc/$pid/cmdline read), the empty expansion produces a malformed two-argument test expression. In busybox ash this happens to evaluate correctly (falls through to export -n "$1=down"), but the unquoted form is fragile. Quote it.

    -

    mwan3: quote $IPv4_REGEX in mwan3_set_user_nft_rule

    +

    mwan3: quote $IPv4_REGEX in mwan3_set_user_nft_rule

    $IPv4_REGEX was unquoted in the grep -qE call on line 951 while the adjacent $IPv6_REGEX on line 950 was correctly quoted. The IPv4 regex contains characters (?, [, ]) that the shell interprets as glob patterns before passing to grep if unquoted. Quote it for consistency and correctness.

    -

    mwan3: fix IPv6 ip rule leak in mwan3_delete_iface_rules

    +

    mwan3: fix IPv6 ip rule leak in mwan3_delete_iface_rules

    mwan3_delete_iface_rules() sets IP="$IP6" for IPv6 interfaces but used bare ip rule list (which defaults to IPv4) to find rule priorities to delete. No IPv6 rules were found, so the for loop never executed and all three ip rules (at priorities IIF_BASE+id, FWMARK_BASE+id, and 3000+id) were leaked on every IPv6 interface ifdown.

    On the next ifup, mwan3_create_iface_rules calls mwan3_delete_iface_rules before adding new rules, but the delete fails for the same reason, so stale rules accumulate on each ifdown/ifup cycle.

    Fix: change ip rule list to $IP rule list so the correct address family is used.

    -

    mwan3: fix dnsmasq_hup running check - json_get_var returns 1 not "true" for boolean true

    +

    mwan3: fix dnsmasq_hup running check - json_get_var returns 1 not "true" for boolean true

    json_get_var stores JSON boolean values as integers (1 for true, 0 for false), not as strings. The previous check [ "$running" = "true" ] never matched, so mwan3_dnsmasq_hup never actually sent SIGHUP to dnsmasq after fw4 reloads. DNS cache was therefore not cleared, leaving stale nftset entries in place.

    mwan3: fix DNAT reply routing broken by misplaced fib-local return rule

    @@ -2320,7 +2645,7 @@ mwan3-lb-test cleanup

    Fix: remove the unguarded fib-local return from its position before the ifaces_in dispatch and reinsert it after, guarded by meta mark & MMX_MASK == 0. Traffic arriving on a mwan3 WAN interface is already marked by the iface_in catchall before reaching this rule, so the guard makes it a no-op for WAN traffic while still blocking non-WAN (LAN, loopback) local-destined traffic from reaching the policy chain. DNAT original packets are now marked by the iface_in catchall, ct mark is saved correctly, and DNAT replies restore it and route back via the correct interface.

    Also remove ct direction reply return from mwan3_output. That rule was added to compensate for ct mark being zero (a consequence of the misplaced fib-local return). With the fib-local return correctly placed and guarded, ct mark is non-zero for inbound WAN connections, the ct mark restore in mwan3_output works correctly, the policy chain guard (meta mark != 0) prevents numgen from firing, and router-level service replies route back via the correct WAN table rather than the main routing table.

    -

    mwan3: add bypass_network UCI option to populate dynamic sets from globals config

    +

    mwan3: add bypass_network UCI option to populate dynamic sets from globals config

    Add mwan3_set_dynamic_network() callback and update mwan3_set_dynamic_sets() to read the bypass_network list from UCI globals config and populate mwan3_dynamic_v4/v6 from it at startup, rather than just flushing the sets empty. Each entry is classified as IPv4 (contains .) or IPv6 (contains :) and added to the appropriate set via the existing nft batch.

    Add mwan3_set_dynamic_sets to the fw4 reload rebuild sequence in both 25-mwan3 and mwan3-fw-rebuild.sh so that bypass_network entries are restored after fw4 reload alongside the connected and custom sets.

    The mwan3_dynamic_v4/v6 sets and mwan3_dynamic chain already existed and were already referenced in mwan3_prerouting, mwan3_output, and mwan3_iface_in_* chains. This change makes the feature accessible via UCI rather than requiring direct nft element injection.

    @@ -2329,14 +2654,14 @@ mwan3-lb-test cleanup

    When fw4 reload occurs during the test wait period, the 25-mwan3 hotplug rebuilds the policy chain without the injected counter rules. Previously this produced TOTAL=unknown and silent per-member zeros with a generic FAIL.

    Now: detect missing counter rules explicitly and report "fw4 reload likely occurred during the test". Also add a human-readable completion timestamp to the summary output.

    -

    luci-app-mwan3: add Bypass networks field to globals page and improve rt_table_lookup UI

    +

    luci-app-mwan3: add Bypass networks field to globals page and improve rt_table_lookup UI

    Add a Bypass networks DynamicList field (UCI option bypass_network) to the globals page. Accepts IPv4 and IPv6 CIDRs directly; traffic to these networks bypasses mwan3 policy routing and uses the default route. Validation uses the cidr datatype with blur-only firing (suppress keyup) via a capture-phase event delegation listener on the container node, covering dynamically added items without a MutationObserver.

    Improve the rt_table_lookup field: rename label from "Routing table lookup" to "Routing table bypass" with a description that explains the bypass effect and that both table numbers and names are accepted. Remove the uinteger datatype constraint which incorrectly rejected table names, since the backend (ip route list table) accepts both.

    -

    18.20 Version 3.3.2

    +

    18.23 Version 3.3.2

    Summary: Version 3.3.2 fixes a spurious tracked-IP entry in ubus status output caused by a naming collision between mwan3track's temporary output file and the TRACK_* glob used by rpcd. The mwan3-lb-test tool gains mandatory client isolation, a Windows test command, and a stale-artifact cleanup subcommand. LuCI receives cross-field family consistency validation in the rule editor, a fix for false "Present (unexpected)" health badges during interface bring-up, source nftset display in the overview rules column, and source nftset support in the traffic path simulator.

    -

    mwan3: rename TRACK_OUTPUT temp file to PING_OUTPUT

    +

    mwan3: rename TRACK_OUTPUT temp file to PING_OUTPUT

    The rpcd status module globs TRACK_* files in each interface's status directory to discover tracked IPs, using the suffix as the IP address. The temporary output file used by mwan3track for ping/httping/nping/nslookup output was named TRACK_OUTPUT, causing the rpcd glob to yield OUTPUT as a spurious tracked IP address in ubus call mwan3 status.

    Rename the variable and its backing file to PING_OUTPUT to avoid the collision with the TRACK_* namespace.

    @@ -2366,7 +2691,7 @@ mwan3-lb-test cleanup

    luci-app-mwan3: add src ipset to overview rules match column

    Display ipset_src (source NFT set) in the Match column of the rules table on the Overview tab. Rename the existing ipset: label to dst ipset: to distinguish it from the new src ipset: label.

    -

    luci-app-mwan3: add ipset_src support to traffic path simulator

    +

    luci-app-mwan3: add ipset_src support to traffic path simulator

    Extend the rule matching simulation to handle the source NFT set option (ipset_src) added alongside the existing destination NFT set (ipset). Updates ruleMatches() to check src_ip membership in the source set, matchSummary() to display it, and the set fetch loop to collect ipset_src names alongside ipset names.

    luci-app-mwan3: improve rule modal family consistency and nftset support

    @@ -2375,14 +2700,14 @@ mwan3-lb-test cleanup

    Source nftset: new ipset_src field allows matching source addresses against an nftset, complementing the existing destination nftset. Both src and dst nftsets can be set on the same rule and are ANDed together.

    Grid display: Source and Destination columns now show the nftset name when no IP address is configured, preventing the rule from appearing as a wildcard match in the listing.

    -

    18.21 Version 3.3.1

    +

    18.24 Version 3.3.1

    Summary: Version 3.3.1 adds source nftset matching (ipset_src) as a complement to the existing destination nftset, fixes three distinct numgen counter contamination bugs that caused load-balancing distributions to skew under inbound or reply traffic, sweeps orphaned policy chains that accumulate when policies are removed from UCI without an fw4 reload, and corrects IPv6 ip rule detection in the routing health check. The release also adds nftset_info as an rpcd ubus method and introduces the mwan3-lb-test CLI tool for verifying load-balancing weight distributions against configured policy members.

    -

    mwan3: add ipset_src option for source nftset matching in rules

    +

    mwan3: add ipset_src option for source nftset matching in rules

    User rules previously supported only a destination nftset match via the ipset UCI option (ip daddr @set). Add ipset_src as an independent source nftset match (ip saddr @set), allowing both src and dst nftsets to be specified on the same rule and ANDed together.

    The existing ipset option is unchanged for backward compatibility. ipset_src follows the same pre-creation and family handling logic as the destination set. The nfproto guard condition is updated to include ipset_src as an implicit family qualifier.

    -

    mwan3: add nftset_info rpcd ubus method

    +

    mwan3: add nftset_info rpcd ubus method

    Returns the name and address-family type of all non-mwan3 nftables sets in table inet fw4. Used by luci-app-mwan3 to annotate the nftset dropdown with family information and validate that a selected set is consistent with the rule's configured address family.

    mwan3: translate icmp to ipv6-icmp for IPv6 family rules

    @@ -2412,14 +2737,14 @@ mwan3-lb-test cleanup

    When a policy is removed from UCI config, its mwan3_policy_* chain persists in table inet fw4 until the next fw4 reload flushes the table. mwan3 restart only manages chains it knows about from current config, leaving orphans indefinitely.

    mwan3_set_policies_nft() now enumerates all mwan3_policy_* chains in the live ruleset and deletes any that have no corresponding UCI policy config before rebuilding the policy chains from config.

    -

    mwan3: fix routing_health ip rule detection for IPv6 interfaces

    +

    mwan3: fix routing_health ip rule detection for IPv6 interfaces

    get_ip_rules() called ip -j rule list which is IPv4-only. mwan3 adds ip rules for IPv6 interfaces via ip -6 rule add, making them invisible to the IPv4 rule list. routing_health() therefore reported iif and fwmark rules as missing for any IPv6 mwan3 interface, showing a false red card even when mwan3 was functioning correctly.

    Fix by querying both -4 and -6 rule tables and merging the results, matching the same pattern already used by get_table_routes(). The stale rule detection also benefits as it now sees IPv6 stale rules too.

    -

    18.22 Version 3.3

    +

    18.25 Version 3.3

    Summary: Version 3.3 adds three major LuCI diagnostic tools - a traffic path Simulator, a static Configuration analyser, and a live Routing health view - backed by two new rpcd ubus methods (nftset_members and routing_health). The configuration analyser detects undefined references, orphaned sections, and rule shadowing including correct IPv6 CIDR containment checks. The routing health view colour-codes per-interface ip rule and routing table state against live kernel state. The apk info vs apk list -I version display bug is also fixed.

    -

    mwan3: add nftset_members and routing_health rpcd ubus methods

    +

    mwan3: add nftset_members and routing_health rpcd ubus methods

    nftset_members { set: "<name>" } returns the current members of a named nft set in table inet fw4. The set name is validated against [a-zA-Z0-9_-]+ before being passed to nft. Used by the luci-app-mwan3 Simulator tab for ipset rule matching and connected-network bypass detection.

    routing_health {} compares the UCI configuration against live kernel state. For each mwan3 interface (by 1-based UCI order index N) it checks for ip rules at priorities 1000+N (iif) and 2000+N (fwmark), checks routing table N for a default route, and reads the mwan3track STATUS file. Reports stale ip rules in mwan3's priority range that have no matching UCI interface. Built-in blackhole (FWMARK_BASE + MAX_IFACES - 2 = 2061) and unreachable (FWMARK_BASE + MAX_IFACES - 1 = 2062) policy rules are explicitly excluded from stale detection.

    Both methods are declared in the luci-app-mwan3 ACL files.

    @@ -2438,7 +2763,7 @@ mwan3-lb-test cleanup

    Routing (Status > MultiWAN Manager > Routing): live comparison of ip rules and routing tables against UCI configuration via routing_health ubus method. Per-interface cards show iif rule, fwmark rule, and routing table state with colour-coded health badges. Includes a field reference explaining the ip rule priority scheme for non-expert users. Stale rule detection excludes built-in blackhole/unreachable policy priorities (2061/2062).

    Also adds nftset_members and routing_health methods to the rpcd module with appropriate ACL entries in both luci-app-mwan3 and luci-app-mwan3-status.

    -

    18.23 Version 3.2.3

    +

    18.26 Version 3.2.3

    Summary: Version 3.2.3 improves tracking status visibility by adding per-IP latency and packet-loss detail to mwan3 status output and fixing the check_quality display to derive its state from mwan3track's runtime files rather than UCI, so changes to UCI without a restart no longer cause the status page to disagree with what is actually running. Stale gateway TRACK_*/LATENCY_*/LOSS_* files from previous PPPoE sessions are cleaned up on each probe list rebuild. The luci-app-mwan3 PKG_VERSION scheme is fixed to prevent apk upgrade from reverting to the official package, and the GitHub Actions APK rename step is corrected to avoid i18n sub-packages overwriting the main package.

    mwan3: add per-IP tracking detail to interfaces status output

    @@ -2449,17 +2774,17 @@ mwan3-lb-test cleanup track 192.168.1.1: ignored

    check_quality is derived from LATENCY_* file presence (not UCI) to reflect actual mwan3track runtime state. The kernel status value "skipped" is displayed as "ignored" for consistency with the LuCI status page.

    -

    mwan3: fix check_quality display inconsistency when UCI is changed without restart

    +

    mwan3: fix check_quality display inconsistency when UCI is changed without restart

    When check_quality is changed in UCI without restarting mwan3track, the rpcd status reported the new UCI value immediately while mwan3track continued running with the old setting, causing the status page to show "Not enabled" for latency/loss even though measurements were still being taken (or vice versa).

    Fix in two parts:

    rpcd: derive check_quality from LATENCY_* file content rather than UCI. mwan3track only writes LATENCY_* files when check_quality=1, so file presence with content is the authoritative indicator of actual runtime state. A fresh read of UCI is not needed on the rpcd side.

    mwan3track: in mwan3_load_track_ips(), re-read check_quality from the current UCI config and remove any stale LATENCY_*/LOSS_* files when it is 0. This runs at startup and on every ifup event, so stale files from a previous check_quality=1 run are cleaned up as soon as the interface next comes up with check_quality=0`, ensuring the rpcd file-based detection also returns the correct result after restart.

    -

    mwan3: remove stale gateway TRACK_/LATENCY_/LOSS_* files on probe list rebuild

    +

    mwan3: remove stale gateway TRACK_/LATENCY_/LOSS_* files on probe list rebuild

    When track_gateway=1 and the gateway IP changes (e.g. PPPoE reconnect), mwan3_load_track_ips builds a new probe list with the new gateway but leaves TRACK_*, LATENCY_*, and LOSS_* files from the old gateway IP on disk. The rpcd status function globs all TRACK_* files to build the tracking IP list, so stale files from previous gateways appear in ubus call mwan3 status alongside the current ones.

    Fix: after building track_ips, scan the interface directory and delete any TRACK_*/LATENCY_*/LOSS_* files for IPs not in the current probe list. Runs at startup and on every ifup event, covering both initial cleanup of files left by a previous mwan3track instance and gateway changes during operation.

    -

    luci-app-mwan3: set explicit PKG_VERSION and fix release workflow

    +

    luci-app-mwan3: set explicit PKG_VERSION and fix release workflow

    Without explicit versioning, luci.mk auto-generates the package version from git commit metadata. In the GitHub Actions SDK build environment the feed directory has no .git (excluded by rsync), so git walks up to the SDK repo and picks up the SDK commit hash and timestamp. The resulting version (e.g. 26.099.76803~8a085e7) is indistinguishable from the official OpenWrt package version format, causing apk to install the official repo version.

    Set PKG_VERSION to $(PKG_SRC_PREFIX).$(PKG_SRC_SUFFIX) where PKG_SRC_PREFIX is derived from the build year and a fixed day value of 999 (not a valid calendar day), ensuring it is always higher than any real date-based official package version for that year. PKG_SRC_SUFFIX encodes the mwan3 version number for readability.

    Also fix the APK rename step to only match the main luci-app-mwan3 package (pattern: PKG_NAME[_-][0-9]*.ext) instead of all APKs in the feed directory. The previous pattern matched all i18n sub-packages as well; since they all got renamed to the same target filename, the last one alphabetically clobbered the main package and was what actually got uploaded to the release.

    @@ -2472,19 +2797,19 @@ mwan3-lb-test cleanup
  • Interface disabled: tracker status shows 'Disabled' (muted) instead of 'Down', and both latency and loss show -
    • -

    18.24 Version 3.2.2

    +

    18.27 Version 3.2.2

    Summary: Version 3.2.2 fixes two misrouting bugs: duplicate jump rules accumulating from repeated fw4 reload cycles caused iface_in chain deletion to fail with "Resource busy", and the unguarded catchall rule in each mwan3_iface_in_* chain was stamping IPv6 packets with the IPv4 interface mark on dual-stack physical devices, breaking QUIC/HTTP3 streams that resumed after conntrack expiry. The gateway IP is moved to the front of the tracking probe list so it is always tested. LuCI receives a visual redesign replacing solid alert cards with bordered flex cards, and adds latency and packet-loss columns to the tracking IP table.

    -

    mwan3: fix iface_in jump rule deletion failing with multiple handles

    +

    mwan3: fix iface_in jump rule deletion failing with multiple handles

    mwan3_delete_iface_nft used a single-shot handle lookup to remove the jump rule for an interface from mwan3_ifaces_in. If repeated fw4 reload cycles caused duplicate jump rules to accumulate, the handle variable received multiple space-separated values and the resulting nft command was syntactically invalid. The chain delete that followed then failed with "Resource busy" since the jump rule was still present.

    Replace the single-shot lookup with a while loop using head -n1, matching the pattern already used for SNAT rule deletion in the same function. This correctly removes all copies of the jump rule one at a time and self-heals any pre-existing duplicates on the next ifdown.

    -

    mwan3: fix iface_in catchall misclassifying IPv6 on shared-device interfaces

    +

    mwan3: fix iface_in catchall misclassifying IPv6 on shared-device interfaces

    The catchall rule in each mwan3_iface_in_* chain lacked a meta nfproto filter. When an IPv4 and IPv6 mwan3 interface share the same physical device (e.g. a dual-stack PPPoE or L2TP WAN), mwan3_ifaces_in dispatches to the IPv4 chain first. The IPv4-specific ip saddr bypass rules do not match IPv6 packets, but the unguarded catchall does, stamping incoming IPv6 packets with the IPv4 interface mark instead of the IPv6 mark.

    The ct mark is then saved with the wrong value. Subsequent packets for that connection restore the IPv4 mark, find no matching ip -6 rule, and fall to the main routing table. For established TCP connections the bug is invisible because ct mark is set on the first outbound packet before any inbound packet arrives. For QUIC (UDP), conntrack entries expire after ~120-180s of inactivity; when the server resumes first the entry has ct mark 0, the catchall fires, and the connection is disrupted - manifesting as stalled streams or a gray page on sites using HTTP/3.

    Fix by adding meta nfproto ipv4/ipv6 to the catchall based on interface family, consistent with the protocol filters already applied to the connected/custom/dynamic bypass rules above it.

    -

    mwan3: ping gateway IP first when track_gateway=1

    +

    mwan3: ping gateway IP first when track_gateway=1

    mwan3_load_track_ips() previously appended the gateway IP to the end of the track_ips list. Since mwan3track stops probing once host_up_count reaches the reliability threshold, the gateway would be skipped whenever an earlier IP responded.

    Prepend the gateway instead so it is always probed first. This gives the gateway probe priority and ensures its reachability is tested on every round regardless of reliability setting.

    @@ -2503,7 +2828,7 @@ mwan3-lb-test cleanup

    The rpcd mwan3 status now returns latency, packetloss and check_quality fields per interface. Extend the Status tab tracking IP table with Latency and Packet Loss columns.

    When check_quality is disabled (the default), the columns display "Not enabled" in muted text rather than zeroed values.

    -

    18.25 Version 3.2.1

    +

    18.28 Version 3.2.1

    Summary: Version 3.2.1 fixes policy status reporting to include all members with their live traffic share percentages (not just the currently-routing member), replaces killall -HUP dnsmasq with a procd-aware targeted SIGHUP to avoid crashing instances still in the startup phase, adds the installed mwan3 package version to mwan3 internal output, and redesigns the LuCI status pages with structured collapsible sections and an IPv6 troubleshooting pane. LuCI also exposes the snat6 IPv6 SNAT option on interface configuration.

    mwan3: show mwan3 package version in internal troubleshooting output

    @@ -2529,7 +2854,7 @@ mwan3-lb-test cleanup

    luci-app-mwan3: expose mwan3 snat6 option for IPv6 interfaces

    Adds an "IPv6 SNAT" form field to the interface configuration modal, visible only when family is set to ipv6. Mirrors the placement of the existing "Track gateway" field. Accepts unset/0 (off, default), 1 (SNAT to the interface's primary GUA), or a literal IPv6 address for NPTv6-style fixed-source pinning. The control is opt-in by design -- see the mwan3 package for the rationale (RFC 6724 source-address selection, NAT66 harm in PA/ULA designs, fixed-saddr upstream requirements).

    -

    18.26 Version 3.2

    +

    18.29 Version 3.2

    Summary: Version 3.2 adds two significant features. First, opt-in per-interface IPv6 SNAT via the snat6 UCI option, which corrects BCP38/uRPF drops for router-originated traffic rerouted by mwan3_output onto a different WAN than the kernel initially selected at sendto(). Second, non-destructive vmap-dispatch mark save/restore: 126 per-mark OR-immediate setter chains replace the previous unmasked connmark operations, making mwan3 fully order-independent with respect to pbr and other fwmark-using packages without requiring coordinated chain priority ordering.

    mwan3: add opt-in IPv6 postrouting SNAT via snat6 UCI option

    @@ -2542,7 +2867,7 @@ mwan3-lb-test cleanup

    Because the save and restore operations are now non-destructive, mwan3's fwmark bits are ORed in and out of the conntrack mark without disturbing bits owned by other packages. This makes mwan3 order-independent with respect to other fwmark-using packages such as pbr, regardless of chain priority ordering.

    The same vmap-dispatch primitive is reused by mwan3_create_policies_nft for load-balancing numgen rules and by the sticky implementation in mwan3_set_user_nft_rule, both of which were previously destructive in unmasked bits. Sticky routing is rebuilt around per-(rule, family, member) ip-only sets that pair with the per-mark setter chains, replacing the legacy ip->mark map. mwan3_delete_iface_map_entries and mwan3_report_policies are updated for the new sticky and numgen forms.

    -

    18.27 Version 3.1.4

    +

    18.30 Version 3.1.4

    Summary: Version 3.1.4 fixes interoperability with pbr by moving mwan3's prerouting and output chains from priority mangle + 1 to mangle - 1, so mwan3 restores and saves its ct mark bits before pbr injects its own marks at mangle priority. With the previous ordering pbr's marks were zeroed before the routing decision and its ip rules never matched.

    mwan3: fix pbr interoperability by running at priority mangle - 1

    @@ -2551,7 +2876,7 @@ mwan3-lb-test cleanup

    With mangle - 1, mwan3 restores and saves its mark before pbr runs. pbr then adds its bits on top, and both sets of marks are present at the routing decision -- matching the coexistence behaviour of the original iptables implementation.

    Add postinst migration to flush and delete the old chains on upgrade, since nftables rejects a chain redeclaration with a different priority.

    -

    18.28 Version 3.1.3

    +

    18.31 Version 3.1.3

    Summary: Version 3.1.3 fixes three status and policy rendering bugs: single-member policies were emitting spurious "unreachable" entries because the empty-string guard on mwan3_mark_to_name never matched; mixed IPv4/IPv6 policies lost one family's members because both shared a single reset list; and equal-weight load-balancing entries were invisible in mwan3 status because nft normalises single-element numgen ranges to plain values that the reporting regex did not match.

    mwan3: fix status reporting for single-member and mixed-family policies

    @@ -2566,10 +2891,10 @@ mwan3-lb-test cleanup

    nft normalizes single-element ranges (e.g. 0-0) to plain values (e.g. 0) when listing rules. The reporting regex only matched the range format N-M : 0xMARK, so equal-weight load balancing entries (weight=1, displayed as N : 0xMARK) were silently skipped, causing the policy to appear empty in mwan3 status despite working correctly.

    Handle both N-M : 0xMARK (weight>1, range preserved by nft) and N : 0xMARK (weight=1, normalized by nft) formats.

    -

    18.29 Version 3.1.2

    +

    18.32 Version 3.1.2

    Summary: Version 3.1.2 improves mwan3rtmon with two fixes: an in-memory route cache replaces the per-event RTM_GETROUTE dump for O(1) ECMP path checks, and a ucode-mod-rtnl double-destructor bug that caused a reliable segfault on clean shutdown is eliminated by letting the GC collect the route listener rather than calling close() explicitly.

    -

    mwan3rtmon: replace route_still_exists() with an in-memory route cache

    +

    mwan3rtmon: replace route_still_exists() with an in-memory route cache

    route_still_exists() issued a full RTM_GETROUTE dump on every route-delete event to check whether an ECMP path still existed before removing per-interface table entries. This is unnecessary overhead.

    Replace with main_route_cache: a { route_key: count } map built from the initial route snapshot in populate_iface_routes() and maintained incrementally in handle_route_event(). The ECMP check becomes an O(1) cache lookup with no rtnl round-trip.

    @@ -2577,7 +2902,7 @@ mwan3-lb-test cleanup

    Calling route_listener.close() explicitly zeroes the resource data pointer while the ucode variable still holds a live reference. When that reference is later released at scope exit, uc_nl_listener_free() fires a second time with arg=NULL and reads uc_nl_listener_t.index at NULL+0x10, producing a reliable "segfault at 10" on every clean shutdown.

    Fix: omit the explicit close() call and let the GC collect the listener naturally. The destructor then fires exactly once with a valid pointer.

    -

    18.30 Version 3.1.1

    +

    18.33 Version 3.1.1

    Summary: Version 3.1.1 is a broad mwan3track hardening release: the disconnecting threshold is raised to suppress false alarms from single transient ping losses, an exclusive flock prevents ghost duplicate tracker processes per interface, sockopt_wrap replaces exit() with graceful error returns so a stale source IP or disappearing interface does not abruptly terminate the tracked process, per-host failure logs are suppressed when the reliability threshold is still met, and interface events are processed at the top of the main loop before pinging to avoid a spurious disconnecting state on wakeup from disabled. LuCI adds a track_gateway checkbox to the interface modal and clarifies the flush_conntrack help text.

    mwan3track: raise disconnecting threshold and log recovery

    @@ -2587,7 +2912,7 @@ mwan3-lb-test cleanup

    mwan3track: use flock to prevent ghost processes for same interface

    Use an exclusive flock on a per-interface lock file to ensure only one mwan3track instance runs per interface at a time. The lock is held for the lifetime of the process and released unconditionally on exit.

    -

    mwan3: sockopt_wrap: replace exit() with graceful error returns

    +

    mwan3: sockopt_wrap: replace exit() with graceful error returns

    A LD_PRELOAD shim must not call exit() on recoverable errors; doing so terminates the tracked process abruptly with no log from mwan3track, making it indistinguishable from a genuine connectivity failure.

    Three cases are fixed:

      @@ -2604,10 +2929,10 @@ mwan3-lb-test cleanup

      When USR2 (ifup) wakes mwan3track from disabled state, the main loop resumes past the MAX_SLEEP wait and immediately starts a ping round before reaching the IFUP_EVENT handler at the bottom of the loop. At that point DEVICE is still stale (empty string at first start), so sockopt_wrap skips SO_BINDTODEVICE, the unbound ping fails, and a spurious "disconnecting" state is logged.

      Fix this by checking IFDOWN_EVENT/IFUP_EVENT at the top of the loop, before any pinging, and using continue to restart the iteration cleanly after firstconnect() has refreshed DEVICE and SRC_IP. The existing handlers at the bottom of the loop are retained for events that arrive during a ping round or the inter-round sleep.

      -

      luci-app-mwan3: clarify flush_conntrack help text

      +

      luci-app-mwan3: clarify flush_conntrack help text

      Update the help text for the flush_conntrack option to clarify that it flushes the entire global conntrack table, and that per-interface conntrack entries are already flushed automatically on ifdown since commit ("mwan3: selectively flush conntrack entries for failed WAN interface on ifdown").

      -

      luci-app-mwan3: add track_gateway option to interface settings

      +

      luci-app-mwan3: add track_gateway option to interface settings

      Add a "Track gateway" checkbox to the interface configuration modal, visible only when the internet protocol is set to IPv4. This exposes the track_gateway UCI option added to the mwan3 backend for automatic point-to-point peer/gateway tracking.

      19. Specific use-cases