🛸 Sync 2026-08-18 08:57:58

luci-app-mptcp/root/usr/libexec/rpcd/luci.mptcp

@@ -51,6 +51,369 @@ local function bwc_rows(dev)
 	return {}
 end
 
+local IP_BIN = "/sbin/ip"
+
+local function trim(s)
+	return (tostring(s or ""):gsub("^%s+", ""):gsub("%s+$", ""))
+end
+
+local function read_proc(path)
+	local f = io.open(path, "r")
+	if not f then return nil end
+	local v = f:read("*l")
+	f:close()
+	if v == nil then return nil end
+	return trim(v)
+end
+
+-- First field of /proc/uptime (seconds since boot), used to turn raw
+-- cumulative MPTcpExt counters into an "events/hour since boot" rate --
+-- a bare cumulative count (e.g. "7021 resets") doesn't say whether that's
+-- alarming or a trickle, since it doesn't say over what timespan.
+local function uptime_seconds()
+	local raw = read_proc("/proc/uptime")
+	local secs = raw and raw:match("^(%S+)")
+	return secs and tonumber(secs) or nil
+end
+
+-- -------------------------------------------------------------------------
+-- MPTCP Diagnostics: explain *why* MPTCP might not be aggregating WANs
+-- correctly -- kernel-level MPTCP counters (fallback to plain TCP,
+-- blackhole/stale-subflow detection, resets, checksum errors, JOIN
+-- failures, ...), the live endpoint/limits state, and a per-WAN cross-check
+-- of whether each multipath-enabled interface actually has a registered
+-- MPTCP endpoint -- turned into a plain-English issue list.
+--
+-- All the underlying commands (multipath -k/-m/-c, ip mptcp ...) are
+-- read-only/local and near-instant, so none of them are timeout-wrapped.
+-- NOTE: never wrap "ip" in "timeout" here -- on this router's busybox
+-- userland, "timeout N ip ..." has been seen to silently dispatch
+-- busybox's own crippled "ip" applet instead of the real /sbin/ip, which
+-- has no "mptcp" subcommand at all. Always call /sbin/ip directly.
+-- -------------------------------------------------------------------------
+
+-- Kernel MPTCP sysctl snapshot (works across the legacy out-of-tree sysctl
+-- layout AND the current in-kernel net.mptcp.* layout -- whichever files
+-- actually exist on this kernel).
+function kernel_settings()
+	local settings = {}
+	settings.enabled = read_proc("/proc/sys/net/mptcp/mptcp_enabled")
+		or read_proc("/proc/sys/net/mptcp/enabled")
+	settings.checksum = read_proc("/proc/sys/net/mptcp/mptcp_checksum")
+		or read_proc("/proc/sys/net/mptcp/checksum_enabled")
+	settings.path_manager = read_proc("/proc/sys/net/mptcp/mptcp_path_manager")
+		or read_proc("/proc/sys/net/mptcp/pm_type")
+	settings.scheduler = read_proc("/proc/sys/net/mptcp/mptcp_scheduler")
+		or read_proc("/proc/sys/net/mptcp/scheduler")
+	settings.syn_retries = read_proc("/proc/sys/net/mptcp/mptcp_syn_retries")
+	settings.stale_loss_cnt = read_proc("/proc/sys/net/mptcp/stale_loss_cnt")
+	settings.add_addr_timeout = read_proc("/proc/sys/net/mptcp/add_addr_timeout")
+	settings.close_timeout = read_proc("/proc/sys/net/mptcp/close_timeout")
+	settings.blackhole_timeout = read_proc("/proc/sys/net/mptcp/blackhole_timeout")
+	settings.allow_join_initial_addr_port = read_proc("/proc/sys/net/mptcp/allow_join_initial_addr_port")
+	return settings
+end
+
+-- The kernel's MPTcpExt* MIB counters, either as the "nstat -z" layout
+-- (name, value, rate -- one counter per line, what "multipath -m" prints)
+-- or the SNMP-style two-line layout (a header line of field names followed
+-- by a value line, both prefixed "MPTcpExt:", found in /proc/net/netstat
+-- and the legacy /proc/net/mptcp_net/snmp). Parse both.
+function parse_counters(raw)
+	local counters = {}
+	local header
+
+	for line in (raw or ""):gmatch("[^\n]+") do
+		-- The nstat-style counter name always continues with a letter right
+		-- after the "MPTcpExt" prefix (e.g. "MPTcpExtMPCapableSYNTX"); the
+		-- SNMP-style header/value lines start "MPTcpExt:" instead. Requiring
+		-- a letter here keeps the two forms from being confused with each
+		-- other (a bare "MPTcpExt:" line's second "word" can itself be a
+		-- number, which would otherwise wrongly match as a counter).
+		local name, value = line:match("^%s*(MPTcpExt%a%w*)%s+(%-?%d+)")
+		if name and value then
+			counters[name:gsub("^MPTcpExt", "")] = tonumber(value)
+		elseif line:match("^MPTcpExt:") then
+			-- SNMP-style: first "MPTcpExt:" line is field names, the next is values
+			if not header then
+				header = {}
+				for field in line:gmatch("%S+") do header[#header + 1] = field end
+			else
+				local i = 0
+				for field in line:gmatch("%S+") do
+					i = i + 1
+					local key = header[i]
+					if key and key ~= "MPTcpExt:" then
+						counters[key] = tonumber(field)
+					end
+				end
+			end
+		end
+	end
+	return counters
+end
+
+-- Where to read the MPTcpExt* counters from for Diagnostics. Deliberately
+-- NOT "multipath -m" here: that wraps "nstat -z", and nstat (without -a)
+-- reports the delta since the *previous* nstat invocation by anyone on the
+-- box (it keeps a shared per-uid history file) -- confirmed live: two
+-- consecutive calls a couple seconds apart came back ~0, while "nstat -a -z"
+-- (ignore history) matched /proc/net/netstat exactly. A page that polls
+-- every 15s and calls that would show "since last poll" numbers while
+-- labelling them "since boot", and would race with anything else on the
+-- router that also calls nstat. Reading the kernel's own procfs SNMP-style
+-- counters directly sidesteps all of that -- always the true cumulative
+-- count, no shared state with other callers.
+function mptcp_counters_raw()
+	local raw = exec_output("cat /proc/net/mptcp_net/snmp 2>/dev/null")
+	if raw == "" then
+		raw = exec_output("grep '^MPTcpExt:' /proc/net/netstat 2>/dev/null")
+	end
+	return raw
+end
+
+-- "ip mptcp endpoint show": one line per registered endpoint, e.g.
+--   10.0.0.2 id 1 subflow dev eth0
+--   88.1.2.3 id 2 signal dev wwan0
+--   10.0.0.5 id 3 subflow backup fullmesh dev eth1
+function parse_endpoints(raw)
+	local endpoints = {}
+	for line in (raw or ""):gmatch("[^\n]+") do
+		local id = line:match("id%s+(%d+)")
+		local dev = line:match("dev%s+(%S+)")
+		local addr = line:match("^%s*(%S+)")
+		if id and addr and addr ~= "id" then
+			endpoints[#endpoints + 1] = {
+				id      = tonumber(id),
+				address = addr,
+				dev     = dev or "",
+				signal  = line:match("%f[%a]signal%f[%A]") ~= nil,
+				subflow = line:match("%f[%a]subflow%f[%A]") ~= nil,
+				backup  = line:match("%f[%a]backup%f[%A]") ~= nil,
+				fullmesh = line:match("%f[%a]fullmesh%f[%A]") ~= nil,
+			}
+		end
+	end
+	return endpoints
+end
+
+-- "ip mptcp limits show" -> "add_addr_accepted 8 subflows 8"
+function parse_limits(raw)
+	local limits = {}
+	for key, value in (raw or ""):gmatch("(%a[%w_]*)%s+(%d+)") do
+		limits[key] = tonumber(value)
+	end
+	return limits
+end
+
+-- Per-WAN endpoint cross-check: for every interface the router considers
+-- multipath-enabled, is its device actually registered as an MPTCP
+-- endpoint? If not, that WAN can never carry an MPTCP subflow no matter
+-- what the schedulers/counters say.
+function wan_endpoint_status(endpoints)
+	local wans = {}
+	uci:foreach("network", "interface", function(s)
+		local name = s[".name"]
+		if name == "loopback" then return end
+
+		local multipath = s["multipath"]
+		if multipath == nil or multipath == "" then
+			multipath = uci:get("openmptcprouter", name, "multipath")
+		end
+		multipath = multipath or "off"
+		if multipath == "off" then return end
+
+		local dev = get_device(name)
+		if dev == "" then dev = s["device"] or s["ifname"] or "" end
+
+		local has_endpoint = false
+		local endpoint_flags = ""
+		for _, ep in ipairs(endpoints) do
+			if dev ~= "" and ep.dev == dev then
+				has_endpoint = true
+				local flags = {}
+				if ep.signal then flags[#flags + 1] = "signal" end
+				if ep.subflow then flags[#flags + 1] = "subflow" end
+				if ep.backup then flags[#flags + 1] = "backup" end
+				if ep.fullmesh then flags[#flags + 1] = "fullmesh" end
+				endpoint_flags = table.concat(flags, ",")
+				break
+			end
+		end
+
+		wans[#wans + 1] = {
+			name           = name,
+			device         = dev,
+			multipath      = multipath,
+			has_endpoint   = has_endpoint,
+			endpoint_flags = endpoint_flags,
+		}
+	end)
+	return wans
+end
+
+-- Turn everything above into a plain-English issue list.
+local function add_issue(issues, severity, code, message, detail)
+	issues[#issues + 1] = {
+		severity = severity,
+		code     = code,
+		message  = message,
+		detail   = detail or "",
+	}
+end
+
+function build_issues(settings, counters, wans)
+	local issues = {}
+
+	if settings.enabled == "0" then
+		add_issue(issues, "error", "mptcp_disabled",
+			"MPTCP is disabled in the kernel.",
+			"No connection can use more than one WAN until MPTCP is re-enabled.")
+	end
+
+	for _, wan in ipairs(wans) do
+		if wan.device == "" then
+			add_issue(issues, "warning", "wan_no_device",
+				string.format("%s: multipath is enabled but the interface has no device yet.", wan.name),
+				"The interface may still be coming up (no IP/link yet).")
+		elseif not wan.has_endpoint then
+			add_issue(issues, "error", "wan_no_endpoint",
+				string.format("%s (%s): no MPTCP endpoint registered for this WAN.", wan.name, wan.device),
+				"This WAN will never be used for an MPTCP subflow, regardless of scheduler/weight settings. " ..
+				"Usually fixed by a hotplug/interface restart (which re-runs 'multipath <dev> on|backup'), " ..
+				"or by checking that the interface actually has a global (non-private-only, non-CGNAT) IP.")
+		end
+	end
+
+	local fallback = (counters.MPCapableFallbackSYNACK or 0) + (counters.MPCapableFallbackACK or 0)
+	if fallback > 0 then
+		add_issue(issues, "warning", "mptcp_fallback",
+			string.format("MPTCP fell back to plain TCP %d time(s).", fallback),
+			"A device on the path (firewall/middlebox/some ISP CGNAT boxes) is stripping the MPTCP TCP option " ..
+			"from the SYN or SYN/ACK, so the kernel gave up and used regular single-path TCP for those " ..
+			"connections. Use the MPTCP Support Check page (tracebox) per-WAN to confirm which WAN(s) strip it.")
+	end
+
+	local stale, recover = counters.SubflowStale or 0, counters.SubflowRecover or 0
+	if stale > 0 then
+		local stuck = stale - recover
+		if stuck > 0 then
+			add_issue(issues, "error", "mptcp_blackhole",
+				string.format("%d subflow(s) currently marked stale (possible blackhole) out of %d total.", stuck, stale),
+				"A path stopped acknowledging data for one or more subflows; the kernel is avoiding it. " ..
+				"If this keeps climbing on one specific WAN, that WAN's path is likely being blackholed.")
+		else
+			add_issue(issues, "warning", "mptcp_blackhole_recovered",
+				string.format("%d subflow(s) were marked stale but all recovered.", stale),
+				"Transient -- a path briefly stopped acking data and the kernel recovered it automatically.")
+		end
+	end
+
+	local hmac_fail = (counters.MPJoinSynAckHMacFailure or 0) + (counters.MPJoinAckHMacFailure or 0)
+	if hmac_fail > 0 then
+		add_issue(issues, "warning", "mptcp_join_hmac_failure",
+			string.format("%d MPTCP JOIN handshake authentication failure(s).", hmac_fail),
+			"A new subflow's JOIN handshake failed HMAC verification -- either packets were altered in " ..
+			"transit or the two ends disagree on the connection's key/token.")
+	end
+
+	if (counters.MPJoinNoTokenFound or 0) > 0 then
+		add_issue(issues, "warning", "mptcp_join_no_token",
+			string.format("%d subflow JOIN attempt(s) referenced an unknown token.", counters.MPJoinNoTokenFound),
+			"The peer (usually the VPS) no longer recognizes the MPTCP connection -- often because its proxy " ..
+			"process restarted and lost state. A new subflow can't attach to a connection the peer forgot.")
+	end
+
+	if (counters.DataCsumErr or 0) > 0 then
+		add_issue(issues, "warning", "mptcp_checksum_error",
+			string.format("%d MPTCP data checksum error(s).", counters.DataCsumErr),
+			"Check that 'mptcp_checksum' matches on both the router and the VPS -- a mismatch (or a path " ..
+			"that corrupts data) triggers this.")
+	end
+
+	-- Split by direction, not just summed: which side is doing the closing
+	-- matters. Confirmed live on the bench (2026-08-16) -- a steady climb
+	-- here traced back to OMR's own per-WAN shadowsocks health-check loop
+	-- (omr-tracker-ss) opening a fresh MPTCP connection through the
+	-- mptcp-enabled sslocal proxy every ~10s and closing it abortively, NOT
+	-- to path interference or the VPS dropping the tunnel -- confirmed by
+	-- MPRstRx staying ~0 while MPRstTx climbed, and by reproducing an
+	-- MPRstTx increment live by firing that exact health-check request.
+	local reset_tx = (counters.MPRstTx or 0) + (counters.MPFastcloseTx or 0)
+	local reset_rx = (counters.MPRstRx or 0) + (counters.MPFastcloseRx or 0)
+	local resets = reset_tx + reset_rx
+	if resets > 0 then
+		local rate_suffix = ""
+		local uptime = uptime_seconds()
+		if uptime and uptime >= 60 then
+			rate_suffix = string.format(" (~%.1f/hour since boot)", resets / (uptime / 3600))
+		end
+		if reset_rx == 0 or reset_tx >= reset_rx * 3 then
+			-- Tx-heavy: this router is doing almost all the closing and the
+			-- peer is barely ever resetting back. That's the signature of a
+			-- local abortive close (e.g. a health-check script tearing down
+			-- its own short-lived proxied connection), not the peer/path
+			-- forcing the connection shut.
+			add_issue(issues, "warning", "mptcp_resets",
+				string.format("%d MPTCP-level reset/fastclose event(s) sent by this router%s.", reset_tx, rate_suffix),
+				"Almost all of these were sent by this router, not the peer (received resets are near zero) -- " ..
+				"that points at something local doing an abortive close of its own MPTCP connections (a common " ..
+				"cause: a health-check/keepalive script that opens a proxied connection, makes one quick request, " ..
+				"then closes it hard) rather than active interference on the path or the VPS dropping the tunnel. " ..
+				"Only worth escalating if it's paired with rising SubflowStale/Blackhole counters above.")
+		else
+			add_issue(issues, "warning", "mptcp_resets",
+				string.format("%d MPTCP-level reset/fastclose event(s)%s.", resets, rate_suffix),
+				"One side force-closed the whole MPTCP connection (all subflows at once) rather than a single " ..
+				"subflow -- can indicate active interference or the proxy/VPN tunnel dropping abruptly.")
+		end
+	end
+
+	local addr_drop = (counters.AddAddrDrop or 0) + (counters.RmAddrDrop or 0)
+	if addr_drop > 0 then
+		-- A handful of drops only matters relative to how much ADD_ADDR/RM_ADDR
+		-- traffic there's actually been -- against a large total it's usually a
+		-- one-off from an interface flap or brief restart, already self-healed
+		-- by the next successful signal, not an ongoing problem. Only surface
+		-- this as an issue once drops are a meaningful share of that traffic
+		-- (>1%); with no successful signals at all to compare against, treat
+		-- it as 100% (nothing has gotten through), which is always worth
+		-- surfacing.
+		local addr_total = (counters.AddAddr or 0) + (counters.AddAddrTx or 0)
+			+ (counters.RmAddr or 0) + (counters.RmAddrTx or 0)
+		local addr_drop_pct = addr_total > 0 and (100 * addr_drop / addr_total) or 100
+		if addr_drop_pct > 1 then
+			local ratio_suffix = ""
+			if addr_total > 0 then
+				ratio_suffix = string.format(" (%d of %d ADD_ADDR/RM_ADDR signals, %.3g%%)",
+					addr_drop, addr_total, addr_drop_pct)
+			end
+			add_issue(issues, "warning", "mptcp_addr_signal_dropped",
+				string.format("%d ADD_ADDR/RM_ADDR signal(s) dropped%s.", addr_drop, ratio_suffix),
+				"A WAN address announcement (or removal) didn't make it to the peer, so the peer may never " ..
+				"attempt a subflow toward that WAN. A handful of drops against a much larger signal volume is " ..
+				"usually a one-off from an interface flap or a brief proxy restart, and self-heals on the next " ..
+				"successful signal -- only worth chasing further if this keeps climbing.")
+		end
+	end
+
+	if (counters.MPFailTx or 0) > 0 or (counters.MPFailRx or 0) > 0 then
+		add_issue(issues, "warning", "mptcp_mapping_failure",
+			"MPTCP data mapping failure signalled (MP_FAIL).",
+			"One side couldn't map received data to the right place in the byte stream on a subflow -- " ..
+			"that subflow gets torn down and traffic continues on the others, if any remain.")
+	end
+
+	if #issues == 0 then
+		add_issue(issues, "ok", "no_issues_detected",
+			"No MPTCP fallback, blackhole, reset or checksum issues detected right now.",
+			"This only reflects counters since boot/last reset -- a healthy snapshot doesn't rule out an " ..
+			"intermittent issue on a specific path; keep this page open while reproducing the problem.")
+	end
+
+	return issues
+end
+
 -- -------------------------------------------------------------------------
 -- RPC methods
 -- -------------------------------------------------------------------------
@@ -354,6 +717,36 @@ local methods = {
 			return { output = exec_output(cmd) }
 		end
 	},
+
+	-- Diagnostics: kernel MPTCP counters/endpoints/limits + a per-WAN
+	-- endpoint cross-check, turned into a plain-English issue list. See the
+	-- helpers above ("MPTCP Diagnostics" section) for the actual logic.
+	diagnose = {
+		call = function()
+			local settings   = kernel_settings()
+			local counters   = parse_counters(mptcp_counters_raw())
+			local endpoints  = parse_endpoints(exec_output(IP_BIN .. " mptcp endpoint show 2>/dev/null"))
+			local limits     = parse_limits(exec_output(IP_BIN .. " mptcp limits show 2>/dev/null"))
+			local wans       = wan_endpoint_status(endpoints)
+			local issues     = build_issues(settings, counters, wans)
+
+			local connections_raw = exec_output("multipath -c 2>/dev/null")
+			local established_count = 0
+			for _ in connections_raw:gmatch("[Ee][Ss][Tt][Aa][Bb]") do
+				established_count = established_count + 1
+			end
+
+			return {
+				settings          = settings,
+				counters          = counters,
+				endpoints         = endpoints,
+				limits            = limits,
+				wans              = wans,
+				issues            = issues,
+				established_count = established_count,
+			}
+		end
+	},
 }
 
 -- -------------------------------------------------------------------------