⛅ Sync 2026-09-29 21:45:00

luci-app-homeproxy-pro/root/etc/homeproxy-pro/scripts/subscription/decoder.uc

@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Pure subscription decoder. The fetcher returns raw bytes
+ * and the orchestrator used to inline the JSON / base64 / SIP008
+ * handling. Pulling it out gives the orchestrator a single
+ * `decode(content, log, url)` call and lets us unit-test each input
+ * shape without involving the network or UCI.
+ *
+ * Three input shapes are recognised, in order:
+ *   1. JSON with a top-level 'servers' array (clash proxy-provider)
+ *   2. JSON array directly (the same data, just unwrapped)
+ *   3. Shadowsocks SIP008 JSON: top-level array of objects with
+ *      server+method. Each entry is tagged nodetype='sip008' so the
+ *      orchestrator can dispatch them differently downstream.
+ *
+ * On JSON failure, decode the content as base64 and split on '\n'
+ * to get one share-link URI per line (sing-box subscription format).
+ *
+ * On total failure (no JSON, no base64), return an empty list and
+ * let the caller log the empty result.
+ *
+ * `url` is optional and only used to enrich the parse-failed log
+ * line; the decoder itself never makes a network call.
+ */
+
+'use strict';
+
+import { isEmpty, decodeBase64Str, redactUrl } from '../homeproxy-pro.uc';
+
+export function decode(content, log, url) {
+	if (isEmpty(content))
+		return [];
+
+	let nodes;
+	try {
+		const parsed = json(content);
+		nodes = parsed.servers || parsed;
+
+		/* Shadowsocks SIP008: each entry is a JSON object with
+		 * server + method, not a URI string.
+		 *
+		 * The probe has to check the type first.  It used to be a bare
+		 * `nodes[0].server`, which THROWS in ucode when nodes[0] is a
+		 * string ("left-hand side expression is not an array or
+		 * object") - and the throw was swallowed by the JSON
+		 * try/catch below, so a perfectly valid `{"servers":["vless://…"]}`
+		 * or `["vless://…"]` subscription fell through to the base64
+		 * fallback, failed there too and decoded to an empty list.  Two
+		 * real subscription shapes were silently discarded, and the
+		 * decoder test had recorded that as the contract.  The same
+		 * guard applies inside the loop so a mixed list (SIP008 objects
+		 * plus share links) keeps both halves instead of losing
+		 * everything on the first string. */
+		if (type(nodes[0]) === 'object' && nodes[0].server && nodes[0].method) {
+			for (let i = 0; i < length(nodes); i++)
+				if (type(nodes[i]) === 'object')
+					nodes[i].nodetype = 'sip008';
+		}
+	} catch (e) {
+		/* The URL carries the subscription token and this line lands in
+		 * homeproxy-pro.log, which is world-readable.  fetcher.uc and the
+		 * orchestrator both redact; this path was missed. */
+		const tag = url ? sprintf('for %s, ', redactUrl(url)) : '';
+		log(sprintf('JSON parse failed %strying base64: %s', tag, e.message));
+		const decoded = decodeBase64Str(content);
+		nodes = decoded ? split(trim(decoded), '\n') : [];
+	}
+
+	return nodes;
+};

luci-app-homeproxy-pro/root/etc/homeproxy-pro/scripts/subscription/fetcher.uc

@@ -0,0 +1,37 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Thin wrapper around homeproxy-pro's wGETVerbose that logs
+ * the failure and returns a flat { content, error } shape. The
+ * orchestrator used to inline the empty-content check + log
+ * message right next to the call, which made the loop body hard
+ * to read and conflated the fetch's success/failure decision with
+ * the downstream decode step.
+ *
+ * On success: returns { content: <string>, error: null }.
+ * On failure: returns { content: null, error: <string> }, having
+ * already logged the failure with the URL. The caller can keep
+ * the loop flat: 'if (result.content === null) continue;'.
+ *
+ * No caching, no retry, no backoff - the orchestrator decides
+ * those if it ever needs them. The Fetcher is a single HTTP GET
+ * + error log, nothing more.
+ */
+
+'use strict';
+
+import { isEmpty, redactUrl, wGETVerbose } from '../homeproxy-pro.uc';
+
+export function fetch(url, user_agent, log) {
+	const result = wGETVerbose(url, user_agent);
+	if (isEmpty(result.content)) {
+		/* Redact the URL before logging: the query string is the
+		 * subscription token and homeproxy-pro.log is world-readable on
+		 * a default install. The original URL goes back to the caller
+		 * via result.error so error propagation still works. */
+		log(sprintf('Failed to fetch resources from %s: %s',
+			redactUrl(url), result.error || 'empty response'));
+		return { content: null, error: result.error || 'empty response' };
+	}
+	return { content: result.content, error: null };
+};

luci-app-homeproxy-pro/root/etc/homeproxy-pro/scripts/subscription/filter.uc

@@ -0,0 +1,95 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Pure subscription filter. The orchestrator
+ * (update_subscriptions.uc) used to inline both the keyword
+ * whitelist/blacklist check and the post-parse policy application
+ * (tls.insecure override, protocol_options.packet_encoding for
+ * vless/vmess). They live here so the orchestrator stays focused on
+ * the fetch/cache/repository flow and the pure logic is unit-testable
+ * without UCI access.
+ *
+ * Neither function touches module state: mode / keywords / opts
+ * come in as arguments, and a logger is passed in for the
+ * invalid-regex warning. That makes the module callable from a
+ * plain test stub and means the orchestrator can keep its own
+ * file-writer log() around the call.
+ */
+
+'use strict';
+
+import { isEmpty } from '../homeproxy-pro.uc';
+
+/* Decide whether a node named `name` should be dropped. `mode` is
+ * one of:
+ *   'disabled'  - filter is off, return false (keep)
+ *   'blacklist' - drop if any keyword regex matches
+ *   'whitelist' - drop unless any keyword regex matches
+ *
+ * `keywords` is a list of strings; each is compiled as a regexp.
+ * One bad regex is logged and skipped, not treated as fatal - this
+ * matches the pre-B1 behaviour, where a stray `[` would not block
+ * the whole subscription.
+ *
+ * Returns true when the node should be skipped. */
+export function check(name, mode, keywords, log) {
+	if (isEmpty(name) || mode === 'disabled' || isEmpty(keywords))
+		return false;
+
+	let matched = false;
+	for (let i in keywords) {
+		let patten;
+		try {
+			patten = regexp(i);
+		} catch (e) {
+			log(sprintf('Skipping invalid filter keyword regex: %s.', i));
+			continue;
+		}
+		if (patten && match(name, patten))
+			matched = true;
+	}
+	if (mode === 'whitelist')
+		matched = !matched;
+
+	return matched;
+};
+
+/* Apply the two policy tweaks the orchestrator used to do inline. The subject
+ * is a canonical Node - the same shape parser/normalize.uc produces and
+ * parser/flatten.uc turns back into UCI keys - not the parser's flat
+ * UCI-key dict:
+ *
+ *   - tls.insecure is set when the node has TLS enabled and the user opted in
+ *     via subscription.allow_insecure='1'
+ *   - protocol_options.packet_encoding is set on vless/vmess nodes to the
+ *     subscription's default
+ *
+ * This used to run on the flat dict, which is why the orchestrator applied
+ * policy on one representation and handed a different one to the Repository.
+ * The canonical Node is now the only business object between the parser and
+ * the Repository, and this was the last step still forcing the flat shape.
+ * The tweaks are a property of the subscription, not of the parsed link, so
+ * they deliberately do not take part in the duplicate fingerprint - which is
+ * computed from the parser output before this runs.
+ *
+ * Mutates and returns the node; the return value is the same object as the
+ * input, so callers can chain. */
+export function apply_policy(node, opts) {
+	if (!node)
+		return node;
+
+	if (node.tls && node.tls.enabled === '1' && opts.allow_insecure === '1')
+		node.tls.insecure = '1';
+
+	if (node.type in ['vless', 'vmess']) {
+		/* normalize() always builds protocol_options; a hand-made node
+		 * (a test fixture, a future caller) may not have it, and the flat
+		 * version simply created the key. */
+		if (!node.protocol_options)
+			node.protocol_options = {};
+
+		node.protocol_options.packet_encoding = opts.packet_encoding;
+	}
+
+	return node;
+};

luci-app-homeproxy-pro/root/etc/homeproxy-pro/scripts/subscription/repository.uc

@@ -0,0 +1,331 @@
+/*
+ * SPDX-License-Identifier: GPL-2.0-only
+ *
+ * Every UCI write the
+ * subscription pipeline performs now lives in this module. Three
+ * entry points cover what used to be six uci.set+commit sites in
+ * update_subscriptions.uc plus the existing node add/update/remove
+ * walk:
+ *
+ *   apply_nodes                 - the add / update / remove walk
+ *                                 for the nodes the subscription
+ *                                 fetch returned this run.
+ *   apply_main_node_refs        - main_node / main_udp_node cleanup:
+ *                                 remove stale urltest members, switch
+ *                                 main_node if its target is gone,
+ *                                 reset both to 'nil' when the
+ *                                 subscription has no nodes.
+ *   scrub_stale_urltest_refs    - one-shot pass over every UCI
+ *                                 routing_node that purges dead
+ *                                 urltest_nodes entries left behind by
+ *                                 nodes that have since been removed.
+ *
+ * Writes are driven by canonical Node objects (post- parser/normalize).
+ * UCI is flat, so apply_nodes expands each node with parser/flatten.uc -
+ * and that call is the module's only canonical -> UCI boundary: callers
+ * hand this module canonical Nodes, never flat dicts, and every UCI key
+ * name comes from parser/mapping.uc. The orchestrator
+ * (update_subscriptions.uc) never writes UCI directly; it orchestrates
+ * fetch + parse + normalize + filter + policy + reload and owns the one
+ * commit.
+ *
+ * A2 (single commit): this module mutates the caller's cursor and never
+ * commits. It used to commit at nine sites - once at the end of
+ * apply_nodes, seven inside apply_main_node_refs, and once per scrubbed
+ * routing_node - while update_subscriptions.uc's own comments called
+ * that "the repository's single commit". A crash between two of them
+ * left the nodes written but main_node still pointing at a deleted
+ * section, and there was no single point at which the run became
+ * visible. The transaction boundary is now the caller's one commit,
+ * after every mutation is staged.
+ */
+
+'use strict';
+
+import { md5 } from 'digest';
+
+import { isEmpty } from '../homeproxy-pro.uc';
+
+/* Relative, like config/loader.uc's '../parser/mapping.uc': ucode's
+ * resolver only searches top-level module names in the -L tree, so a
+ * bare 'parser/flatten.uc' does not resolve. */
+import { flatten } from '../parser/flatten.uc';
+
+/* --- apply_nodes -------------------------------------------------------- */
+
+/* Add / update / remove the subscription-owned nodes against UCI.
+ *
+ *   node_cache   - { [groupHash]: { [nameHash]: canonical_node } }
+ *                  A lookup table keyed by subscription URL group
+ *                  and then by the per-node name hash. The fetch
+ *                  loop populates this for de-duplication.
+ *   node_result  - [[canonical_node, ...]] (one inner list per URL)
+ *                  The de-duplicated, policy-applied nodes in the
+ *                  order they should appear in UCI.
+ *
+ * Behaviour parity with the previous code:
+ *   - user-created nodes (no grouphash) are never touched
+ *   - nodes whose subscription has no cache entry (network
+ *     failure for that URL) are left in place; we don't delete
+ *     a node just because today's fetch didn't return it
+ *   - a node in UCI but not in the cache is deleted
+ *   - a node in both has every field of the new config written
+ *     (including options the stored section did not have yet),
+ *     and the options the new config no longer carries are removed
+ *   - new nodes are added under name = md5(groupHash + label), the
+ *     same hash the cache uses, so a subsequent re-fetch recognises
+ *     them as existing
+ *
+ * Returns the { added, removed } counts. No commit here: the caller's
+ * single commit is the boundary of "this run wrote something". */
+function apply_nodes(uci, uciconfig, ucinode, node_cache, node_result, log) {
+	let added = 0, removed = 0;
+	/* Mark the canonical Node objects the foreach loop above has
+	 * already updated in place, so the second loop skips them. We
+	 * use object identity (the same Node object is held in node_cache
+	 * and in node_result, so `seen[node] === node` works as a set),
+	 * not the section name or nameHash: the orchestrator hands the
+	 * Repository the same Node instances, and tying the marker to the
+	 * object means a stale string key cannot match a renamed node. */
+	const seen = {};
+
+	uci.foreach(uciconfig, ucinode, (cfg) => {
+		/* User-created nodes do not have a grouphash. The
+		 * repository must never touch them, even if the
+		 * subscription fetch returns nothing for today. */
+		if (!cfg.grouphash)
+			return null;
+
+		/* No cache entry for this subscription group means
+		 * the fetch failed for that URL. Leave the existing
+		 * nodes in place so the user does not lose them on
+		 * a transient network blip. */
+		if (!node_cache[cfg.grouphash] || length(node_cache[cfg.grouphash]) === 0)
+			return null;
+
+		const incoming = node_cache[cfg.grouphash][cfg['.name']];
+
+		if (!incoming) {
+			uci.delete(uciconfig, cfg['.name']);
+			removed++;
+			log(sprintf('Removing node: %s.', cfg.label || cfg['name']));
+			return null;
+		}
+
+		/* Canonical Node -> flat UCI keys. flatten() drops
+		 * nulls so a freshly-saved canonical Node does not
+		 * litter /etc/config with `tls_sni ''` entries for
+		 * fields the parser never set; the Loader treats null
+		 * and absent the same, so this is observably
+		 * equivalent. */
+		const flat = flatten(incoming);
+
+		/* Write every option the new config carries, including
+		 * ones the stored section did not have yet. Walking only
+		 * the OLD section's keys used to miss new options
+		 * (plugin, tls_sni, packet_encoding) the subscription
+		 * started sending after the node was first imported. */
+		for (let v in keys(flat))
+			uci.set(uciconfig, cfg['.name'], v, flat[v]);
+
+		/* Then drop the options the new config no longer
+		 * carries. `.name` / `.type` / `.index` are section
+		 * metadata, not options; deleting them is meaningless
+		 * at best. */
+		for (let v in keys(cfg)) {
+			if (substr(v, 0, 1) === '.')
+				continue;
+			if (!(v in flat))
+				uci.delete(uciconfig, cfg['.name'], v);
+		}
+
+		seen[incoming] = true;
+	});
+
+	for (let nodes in node_result)
+		map(nodes, (node) => {
+			if (seen[node])
+				return null;
+
+			/* normalize() exposes the source label as `name`; `label`
+			 * is only set when the orchestrator carries it over. Relying
+			 * on `label` alone hashed every node of a subscription to the
+			 * same section name, because undefined stringifies
+			 * identically - four nodes collapsed into one section mixing
+			 * several protocols' options. Accept either. */
+			const label = node.label || node.name;
+			const nameHash = md5(node.grouphash + label);
+
+			/* The one boundary where a canonical Node becomes UCI keys.
+			 *
+			 * UCI is a flat key/value store, so this expansion has to
+			 * happen somewhere; keeping it inside the Repository's write
+			 * path is what lets every other layer - the orchestrator, the
+			 * filter, the policy step - speak canonical only. If a field
+			 * must not reach UCI, or its UCI name must change, this call
+			 * and parser/mapping.uc are the places to look; no caller
+			 * hands this function a flat dict. */
+			const flat = flatten(node);
+
+			uci.set(uciconfig, nameHash, 'node');
+			for (let v in keys(flat))
+				uci.set(uciconfig, nameHash, v, flat[v]);
+
+			added++;
+			log(sprintf('Adding node: %s.', label));
+		});
+
+	return { added, removed };
+}
+
+/* --- apply_main_node_refs ---------------------------------------------- */
+
+/* Prune the entries of an UCI list that no longer point at a live
+ * node. Used twice below for `main_urltest_nodes` and
+ * `main_udp_urltest_nodes`; the helper writes the cleaned list back
+ * only when it actually changed, and returns it for the caller to
+ * inspect (an empty list is the trigger to switch to first_server). */
+function maybeScrubUrltestNodes(uci, uciconfig, ucimain, listOption, log) {
+	const old_list = uci.get(uciconfig, ucimain, listOption) || [];
+	const cleaned = filter(old_list, (v) => {
+		if (!uci.get(uciconfig, v)) {
+			log(sprintf('Node %s is gone, removing from urltest list.', v));
+			return false;
+		}
+		return true;
+	});
+	if (length(cleaned) !== length(old_list))
+		uci.set(uciconfig, ucimain, listOption, cleaned);
+	return cleaned;
+}
+
+/* Switch a main_* field off the dangling reference and onto the first
+ * available node. The two callers below differ only in the UCI option
+ * name (`main_node` vs `main_udp_node`) and the user-facing log line,
+ * so this is what made the pre-refactor code mirror itself.
+ *
+ * `current_value` is what the field currently points at (the user-chosen
+ * section name, the literal 'urltest', or 'same' / 'nil').  We only
+ * overwrite the field when the section named by `current_value` no
+ * longer exists in UCI; the previous version asked `uci.get(uciconfig,
+ * field)` which is "does a section literally named `field` exist" - the
+ * config has cfg0xxxx names, never one named `main_node`, so the check
+ * was always false and the user's pick got silently replaced with
+ * first_server on every subscription update.
+ *
+ * The new value is also pushed onto `result.log` so the orchestrator
+ * can replay it (it uses `result.log` to log the same lines verbatim;
+ * the homeproxy-pro.log capture goes through the separate `log` arg). */
+function switchMainNodeOffDanglingRef(uci, uciconfig, ucimain, field, current_value, first_server, result, log) {
+	if (uci.get(uciconfig, current_value))
+		return current_value;
+	uci.set(uciconfig, ucimain, field, first_server);
+	const msg = sprintf('Main%s node is gone, switching to the first node.',
+		field === 'main_udp_node' ? ' UDP' : '');
+	log(msg);
+	push(result.log, msg);
+	return first_server;
+}
+
+/* Replace the 4 inline uci.set/commit sites (plus the 2-line
+ * 'reset to nil' block) in update_subscriptions.uc. Logic parity:
+ *
+ *   - if main_node is set and at least one subscription node exists:
+ *     * when main_node === 'urltest', scrub main_urltest_nodes down
+ *       to nodes that still exist, then switch main_node to the first
+ *       surviving one if the urltest set is empty
+ *     * otherwise, switch main_node to first_server if its target is
+ *       gone
+ *     * same shape for main_udp_node, only when main_udp_node is set
+ *       AND not 'same'
+ *   - if main_node is set but no subscription nodes exist, reset
+ *     both main_node and main_udp_node to 'nil'
+ *
+ * `ctx` carries the inputs the orchestrator used to read from UCI:
+ *   { main_node, main_udp_node, has_nodes }
+ *
+ * Mutates only, no commit. The orchestrator commits once after
+ * apply_nodes, this method and scrub_stale_urltest_refs have all
+ * staged their changes, so a run is either fully visible or not at all.
+ *
+ * Returns { main_node: <new>, main_udp_node: <new>, log: [..] } so
+ * the orchestrator can log "Main node is gone, switching to ..." /
+ * "No available node, disable tproxy." exactly as before. */
+function apply_main_node_refs(uci, uciconfig, ucimain, ucinode, ctx, log) {
+	/* Object destructuring is not part of the dialect the ucode on the
+	 * target accepts (ImmortalWrt ucode 2026.01.16 rejects
+	 * `const { a } = x` with "Expecting variable name"), so pull the
+	 * fields out by name. tests/ucode/test_ucode_grammar.sh guards this. */
+	const main_node = ctx.main_node,
+	      main_udp_node = ctx.main_udp_node;
+	const result = { main_node, main_udp_node, log: [] };
+
+	if (isEmpty(main_node))
+		return result;
+
+	const first_server = uci.get_first(uciconfig, ucinode);
+	if (!first_server) {
+		/* No nodes left at all: reset both to 'nil'. */
+		uci.set(uciconfig, ucimain, 'main_node', 'nil');
+		uci.set(uciconfig, ucimain, 'main_udp_node', 'nil');
+		result.main_node = 'nil';
+		result.main_udp_node = 'nil';
+		push(result.log, 'No available node, disable tproxy.');
+		return result;
+	}
+
+	if (main_node === 'urltest') {
+		const cleaned = maybeScrubUrltestNodes(uci, uciconfig, ucimain, 'main_urltest_nodes', log);
+
+		if (!length(cleaned))
+			result.main_node = switchMainNodeOffDanglingRef(uci, uciconfig, ucimain, 'main_node', main_node, first_server, result, log);
+	} else {
+		result.main_node = switchMainNodeOffDanglingRef(uci, uciconfig, ucimain, 'main_node', main_node, first_server, result, log);
+	}
+
+	if (!isEmpty(main_udp_node) && main_udp_node !== 'same') {
+		if (main_udp_node === 'urltest') {
+			const cleaned = maybeScrubUrltestNodes(uci, uciconfig, ucimain, 'main_udp_urltest_nodes', log);
+
+			if (!length(cleaned))
+				result.main_udp_node = switchMainNodeOffDanglingRef(uci, uciconfig, ucimain, 'main_udp_node', main_udp_node, first_server, result, log);
+		} else {
+			result.main_udp_node = switchMainNodeOffDanglingRef(uci, uciconfig, ucimain, 'main_udp_node', main_udp_node, first_server, result, log);
+		}
+	}
+
+	return result;
+}
+
+/* --- scrub_stale_urltest_refs ----------------------------------------- */
+
+/* Walk every routing_node and prune the urltest_nodes list to drop
+ * entries that no longer point at a live node. Mutates only: the caller
+ * commits once after this returns. Returns { changed } - the number of
+ * routing_nodes it rewrote - which is what the old `commits` counter
+ * measured (it counted its own commits; the previous code committed
+ * inside the foreach loop, once per scrubbed routing_node). */
+function scrub_stale_urltest_refs(uci, uciconfig, log) {
+	let changed = 0;
+
+	uci.foreach(uciconfig, 'routing_node', (cfg) => {
+		if (cfg.node !== 'urltest' || isEmpty(cfg.urltest_nodes))
+			return null;
+
+		const cleaned_nodes = filter(cfg.urltest_nodes, (v) => uci.get(uciconfig, v));
+		if (length(cleaned_nodes) === length(cfg.urltest_nodes))
+			return null;
+
+		uci.set(uciconfig, cfg['.name'], 'urltest_nodes', cleaned_nodes);
+		changed++;
+		log(sprintf('Routing node %s: removed gone nodes from urltest list.', cfg['.name']));
+	});
+
+	return { changed };
+}
+
+export const Repository = {
+	apply_nodes,
+	apply_main_node_refs,
+	scrub_stale_urltest_refs
+};