🎉 Sync 2026-11-02 12:48:14

safeshield/spec/blocklist_spec.sh

@@ -8,6 +8,27 @@ Describe 'SafeShield blocklist behavior'
 		The error should equal ''
 	End
 
+	It 'builds large lists with bounded merge scratch files and preserves precedence'
+		When call ss_case_blocklist_tmpfs
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
+	It 'preserves the exact legacy domain order for both dnsmasq modes'
+		When call ss_case_blocklist_legacy_order
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
+	It 'filters block rules with bounded-memory lookup and no-allow fast path'
+		When call ss_case_blocklist_low_memory
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
 	It 'resolves and downloads multiple Hub artifact sources'
 		When call ss_case_multi_artifact
 		The status should be success

safeshield/spec/core_spec.sh

@@ -50,6 +50,13 @@ Describe 'SafeShield core shell behavior'
 		The error should equal ''
 	End
 
+	It 'updates status fields in batches rather than one process per key'
+		When call ss_case_status_batch_writes
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
 	It 'removes preserved legacy license keys during upgrade'
 		When call ss_case_legacy_license_cleanup
 		The status should be success

safeshield/spec/lifecycle_spec.sh

@@ -1,7 +1,7 @@
 # shellcheck shell=sh
 
 Describe 'SafeShield lifecycle and refresh state machines'
-	It 'rolls back full refresh failures while preserving the active blocklist for upgrade-required responses'
+	It 'keeps DNS running before installation failures and rolls back post-install failures'
 		When call ss_case_refresh_state_machine
 		The status should be success
 		The output should equal ''

safeshield/spec/locking_spec.sh

@@ -0,0 +1,32 @@
+#!/bin/sh
+# shellcheck shell=sh
+
+Describe 'SafeShield directory locking and state consistency'
+	It 'prevents lock theft, reclaims stale owners and respects ownership'
+		When call ss_case_lock_owner_and_contention
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
+	It 'serializes concurrent operations using BusyBox-compatible locks'
+		When call ss_case_lock_parallel
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
+	It 'serializes status read-modify-write operations under the same lock'
+		When call ss_case_status_write_serialization
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
+	It 'does not corrupt the previous status when serialization fails'
+		When call ss_case_status_write_failure
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+End

safeshield/spec/spec_helper.sh

@@ -16,6 +16,8 @@ export SS_SPEC_ROOT
 # shellcheck disable=SC1091
 . "$SS_SPEC_ROOT/spec/support/lifecycle_cases.sh"
 # shellcheck disable=SC1091
+. "$SS_SPEC_ROOT/spec/support/locking_cases.sh"
+# shellcheck disable=SC1091
 . "$SS_SPEC_ROOT/spec/support/blocklist_cases.sh"
 # shellcheck disable=SC1091
 . "$SS_SPEC_ROOT/spec/support/device_api_cases.sh"

safeshield/spec/statistics_spec.sh

@@ -119,6 +119,13 @@ Describe 'SafeShield statistics behavior'
 		The error should equal ''
 	End
 
+	It 'prunes expired statistics while preserving the short upload window'
+		When call ss_case_statistics_retention_cache
+		The status should be success
+		The output should equal ''
+		The error should equal ''
+	End
+
 	It 'loads the statistics collector from focused awk modules'
 		When call ss_case_statistics_modules
 		The status should be success

safeshield/spec/support/blocklist_cases.sh

@@ -140,6 +140,203 @@ sample-two.example'
 	ss_spec_assert_eq "$DNS_CHECK_COUNT" '2'
 )
 
+# A larger, overlapping fixture exercises sorted streaming, tmpfs cleanup and
+# precedence without downloading any external artifact.
+ss_case_blocklist_tmpfs() (
+	set -eu
+	TMP_DIR="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
+	SS_TMP_DIR="$TMP_DIR/tmp"
+	SS_DNSMASQ_DIR="$TMP_DIR/dnsmasq.d"
+	SS_BLOCKLIST_FILE="$SS_DNSMASQ_DIR/safeshield.blocklist"
+	ss_min_valid_line_count=1
+	ss_max_blocklist_file_size_kb=1024
+	ss_dnsmasq_sanity_check=0
+	mkdir -p "$SS_TMP_DIR" "$SS_DNSMASQ_DIR"
+	ss_should_stop() { [ "${STOP_NOW:-0}" = 1 ]; }
+	ss_status_set() { :; }
+	ss_status_add_error() { :; }
+	log_info() { :; }
+	log_error() { :; }
+	ss_blocklist_tmp_path() { printf '%s/.safeshield.blocklist.tmp.%s\n' "$SS_DNSMASQ_DIR" "$$"; }
+	ss_install_blocklist_atomic() { mv -f "$1" "$2"; }
+	DNSMASQ_PATCHED=1
+	ss_dnsmasq_safeshield_block_supported() { [ "$DNSMASQ_PATCHED" = '1' ]; }
+	# shellcheck disable=SC1091
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
+
+	awk 'BEGIN { for (i = 1; i <= 6000; i++) printf "ads%05d.example\n", i }' >"$SS_TMP_DIR/api.0.block.txt"
+	awk 'BEGIN { for (i = 4000; i <= 10000; i++) printf "ads%05d.example\n", i }' >"$SS_TMP_DIR/api.1.block.txt"
+	printf '%s\n' 'ads00042.example' 'hub-allow.example' >"$SS_TMP_DIR/api.2.allow.txt"
+	printf '%s\n' 'hub-allow.example' 'local.example' >"$SS_TMP_DIR/local.block.txt"
+	printf '%s\n' 'local.example' 'ads00042.example' >"$SS_TMP_DIR/local.allow.txt"
+
+	# Prior implementations left 7 merge scratch files on tmpfs. Even stale
+	# files from a previous run must be released after this refresh.
+	printf 'stale\n' >"$SS_TMP_DIR/effective-api-blocks.domains"
+	printf 'stale\n' >"$SS_TMP_DIR/final-blocks.domains"
+	ss_merge_lists
+	ss_spec_assert_eq "$(grep -c '^safeshield-block=/' "$SS_BLOCKLIST_FILE")" '10000'
+	ss_spec_assert_eq "$(grep -c '^server=/' "$SS_BLOCKLIST_FILE")" '2'
+	ss_spec_assert_file_contains "$SS_BLOCKLIST_FILE" 'safeshield-block=/hub-allow.example/'
+	ss_spec_assert_file_not_contains "$SS_BLOCKLIST_FILE" 'safeshield-block=/ads00042.example/'
+	ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'server=/ads00042.example/#'
+	ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'server=/local.example/#'
+	# Cached source shards and local rules must not be deleted by cleanup.
+	ss_spec_assert_exists "$SS_TMP_DIR/api.0.block.txt"
+	ss_spec_assert_exists "$SS_TMP_DIR/local.block.txt"
+	for derived in \
+		api-blocks.domains api-allows.domains \
+		effective-local-blocks.domains effective-api-allows.domains \
+		effective-allows.domains effective-api-blocks.domains final-blocks.domains; do
+		ss_spec_assert_not_exists "$SS_TMP_DIR/$derived"
+	done
+
+	# Stock dnsmasq rules retain identical precedence and order.
+	DNSMASQ_PATCHED=0
+	ss_merge_lists
+	ss_spec_assert_eq "$(grep -c '^address=/' "$SS_BLOCKLIST_FILE")" '10000'
+	ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'address=/hub-allow.example/#'
+	ss_spec_assert_file_not_contains "$SS_BLOCKLIST_FILE" 'address=/ads00042.example/#'
+	for derived in api-blocks.domains effective-api-blocks.domains final-blocks.domains; do
+		ss_spec_assert_not_exists "$SS_TMP_DIR/$derived"
+	done
+
+	# A cancelled refresh must not install partial rules or retain scratch.
+	before="$(cksum "$SS_BLOCKLIST_FILE")"
+	STOP_NOW=1
+	result=0
+	ss_merge_lists || result=$?
+	ss_spec_assert_eq "$result" '130'
+	ss_spec_assert_eq "$(cksum "$SS_BLOCKLIST_FILE")" "$before"
+	ss_spec_assert_not_exists "$SS_TMP_DIR/api-blocks.domains"
+	ss_spec_assert_not_exists "$SS_TMP_DIR/effective-api-blocks.domains"
+)
+
+# Print an actionable diff if this ordering contract regresses in CI.
+ss_assert_legacy_blocklist_order() (
+	# Use a subshell to keep these variables scoped under POSIX sh.
+	expected="$1"
+	actual_file="$2"
+	expected_file="${actual_file}.expected.$$"
+
+	printf '%s\n' "$expected" >"$expected_file"
+	if ! cmp -s "$expected_file" "$actual_file"; then
+		diff -u "$expected_file" "$actual_file" >&2 || :
+		rm -f "$expected_file"
+		return 1
+	fi
+	rm -f "$expected_file"
+)
+
+# Ordering is part of our byte-for-byte legacy blocklist compatibility.
+# Sorting decorated dnsmasq rule strings changes prefix ordering ("foo" vs
+# "foo-bar" / "foo.bar"); always compare by the undressed domain field.
+ss_case_blocklist_legacy_order() (
+	set -eu
+	TMP_DIR="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
+	SS_TMP_DIR="$TMP_DIR/tmp"
+	SS_DNSMASQ_DIR="$TMP_DIR/dnsmasq.d"
+	SS_BLOCKLIST_FILE="$SS_DNSMASQ_DIR/safeshield.blocklist"
+	ss_min_valid_line_count=0
+	ss_max_blocklist_file_size_kb=1024
+	ss_dnsmasq_sanity_check=0
+	mkdir -p "$SS_TMP_DIR" "$SS_DNSMASQ_DIR"
+	ss_should_stop() { return 1; }
+	ss_status_set() { :; }
+	ss_status_add_error() { :; }
+	log_info() { :; }
+	log_error() { :; }
+	ss_blocklist_tmp_path() { printf '%s/.safeshield.blocklist.tmp.%s\n' "$SS_DNSMASQ_DIR" "$$"; }
+	ss_install_blocklist_atomic() { mv -f "$1" "$2"; }
+	DNSMASQ_PATCHED=1
+	ss_dnsmasq_safeshield_block_supported() { [ "$DNSMASQ_PATCHED" = '1' ]; }
+	# shellcheck disable=SC1091
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
+	# API shards are pre-sorted by ss_download_api_artifacts().  Keep this
+	# fixture representative of the actual -m (merge) precondition.
+	printf '%s\n' 'abc.test' 'foo' 'foo-bar' 'foo.bar' 'foo_bar' |
+		LC_ALL=C sort -u >"$SS_TMP_DIR/api.0.block.txt"
+	printf '%s\n' 'abc.test-more' 'abc.test.uk' |
+		LC_ALL=C sort -u >"$SS_TMP_DIR/api.1.block.txt"
+	ss_merge_lists
+	expected='safeshield-block=/abc.test/
+safeshield-block=/abc.test-more/
+safeshield-block=/abc.test.uk/
+safeshield-block=/foo/
+safeshield-block=/foo-bar/
+safeshield-block=/foo.bar/
+safeshield-block=/foo_bar/'
+	ss_assert_legacy_blocklist_order "$expected" "$SS_BLOCKLIST_FILE"
+	DNSMASQ_PATCHED=0
+	ss_merge_lists
+	expected='address=/abc.test/#
+address=/abc.test-more/#
+address=/abc.test.uk/#
+address=/foo/#
+address=/foo-bar/#
+address=/foo.bar/#
+address=/foo_bar/#'
+	ss_assert_legacy_blocklist_order "$expected" "$SS_BLOCKLIST_FILE"
+)
+
+# Both low-memory filtering paths must retain the legacy allow precedence and
+# the original local-block-first rule stream. The final merge sorts afterwards.
+ss_case_blocklist_low_memory() (
+	set -eu
+	TMP_DIR="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
+	mkdir -p "$TMP_DIR"
+	SS_TMP_DIR="$TMP_DIR"
+	# shellcheck disable=SC1091
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
+	cat >"$TMP_DIR/blocks" <<'DATA'
+ads.example
+foo
+foo-bar
+sub.ads.example
+invalid/path
+CAPS.example
+valid.example
+DATA
+	printf '%s\n' 'local.example' >"$TMP_DIR/local"
+	: >"$TMP_DIR/shadow"
+	ss_write_effective_block_rules \
+		"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
+		"$TMP_DIR/result" 'safeshield-block=/' '/'
+	expected='safeshield-block=/local.example/
+safeshield-block=/ads.example/
+safeshield-block=/foo/
+safeshield-block=/foo-bar/
+safeshield-block=/sub.ads.example/
+safeshield-block=/valid.example/'
+	ss_spec_assert_eq "$(cat "$TMP_DIR/result")" "$expected"
+
+	# A present allow shadows the matching Hub block and its descendants, but
+	# does not remove an explicit local block or unrelated Hub domains.
+	printf '%s\n' 'foo' 'ads.example' >"$TMP_DIR/shadow"
+	ss_write_effective_block_rules \
+		"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
+		"$TMP_DIR/result" 'address=/' '/#'
+	expected='address=/local.example/#
+address=/foo-bar/#
+address=/valid.example/#'
+	ss_spec_assert_eq "$(cat "$TMP_DIR/result")" "$expected"
+
+	# When the local list is empty, the no-allow fast path must still process
+	# Hub block records (including the first record of the second input file).
+	: >"$TMP_DIR/local"
+	: >"$TMP_DIR/shadow"
+	ss_write_effective_block_rules \
+		"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
+		"$TMP_DIR/result" 'address=/' '/#'
+	ss_spec_assert_eq "$(head -n 1 "$TMP_DIR/result")" 'address=/ads.example/#'
+	ss_spec_assert_eq "$(grep -c '^address=/' "$TMP_DIR/result")" '5'
+	BLOCKLIST="$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
+	ss_spec_assert_file_contains "$BLOCKLIST" 'sort -S 1M -u -t/ -k2,2 -o "$final" "$final"'
+)
+
 ss_case_multi_artifact() (
 	set -eu
 	TMP_DIR="$(ss_spec_tmpdir)"

safeshield/spec/support/core_cases.sh

@@ -6,6 +6,7 @@ ss_case_utils() (
 	PKG_NAME='safeshield-test'
 	export PKG_NAME
 	# shellcheck disable=SC1091
+	SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
 	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
 
 	command_exists sh
@@ -25,6 +26,7 @@ ss_case_config_validation() (
 	PKG_NAME='safeshield-test'
 	export PKG_NAME
 	# shellcheck disable=SC1091
+	SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
 	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
 
 	WARNINGS=''
@@ -121,6 +123,7 @@ ss_case_dnsmasq_version() (
 	PKG_NAME='safeshield'
 	export PKG_NAME
 	# shellcheck disable=SC1091
+	SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
 	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
 	DNSMASQ_FEATURE_HEALTH=''
 	ss_status_set() {
@@ -378,6 +381,15 @@ ss_case_status_state() (
 	record() { printf '%s\t%s\t%s\n' "$1" "${2:-}" "${3:-}" >>"$CALLS"; }
 	ss_status_set() { record set "$1" "$2"; }
 	ss_status_add_error() { record error "$1"; }
+	# Capture each value for existing state assertions while recording that a
+	# whole group is now committed through one locked status-store operation.
+	ss_status_set_many() {
+		record batch set_many "$#"
+		while [ "$#" -ge 2 ]; do
+			record set "$1" "$2"
+			shift 2
+		done
+	}
 	ss_status_apply() {
 		record apply "$1" "${2:-}"
 		return 0
@@ -388,6 +400,7 @@ ss_case_status_state() (
 		[ "$1" -ge 0 ] 2>/dev/null
 	}
 	ss_status_mark_failure 'artifact_download_failed'
+	ss_spec_assert_eq "$(grep -c "$(printf 'batch\tset_many')" "$CALLS")" '1'
 	grep -F "$(printf 'set\tstatus\terror')" "$CALLS" >/dev/null
 	grep -F "$(printf 'set\tlast_failure\t1000')" "$CALLS" >/dev/null
 	grep -F "$(printf 'set\tlast_result\terror')" "$CALLS" >/dev/null
@@ -405,6 +418,7 @@ ss_case_status_state() (
 	! grep -F "$(printf 'set\tnext_refresh_at')" "$CALLS" >/dev/null
 	: >"$CALLS"
 	ss_status_reset_health_fields
+	ss_spec_assert_eq "$(grep -c "$(printf 'batch\tset_many')" "$CALLS")" '1'
 	for key in health_dnsmasq_binary health_dns_runtime health_artifact_download dnsmasq_min_version; do
 		grep -F "$(printf 'set\t%s\t' "$key")" "$CALLS" >/dev/null
 	done
@@ -467,8 +481,45 @@ ss_case_status_version() (
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/Makefile" "echo '\$(PKG_VERSION)-r\$(PKG_RELEASE)' > \$(1)/usr/lib/safeshield/version"
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/share/rpcd/ucode/safeshield/core.uc" 'fs.readfile(`/usr/lib/${PKG_NAME}/version`)'
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/share/rpcd/ucode/safeshield/status.uc" 'version: PKG_VERSION,'
-	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'safeshield_upgrade_required'
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'failure_code="$(ss_resolve_error_code)"'
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'failure_code="$(ss_artifact_download_error_code)"'
-	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'if [ "$failure_code" != "safeshield_upgrade_required" ]; then'
+	ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'No blocklist replacement has happened'
+)
+
+# The legacy refresh reset used one ucode process + one locked JSON rewrite
+# per key. Grouping related updates keeps the same values with fewer writes.
+ss_case_status_batch_writes() (
+	set -eu
+	TMP_DIR="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
+	RUNNING_STATUS_FILE="$TMP_DIR/status.json"
+	# shellcheck disable=SC1091
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
+	CALLS="$TMP_DIR/calls"
+	: >"$CALLS"
+	ss_status_apply() {
+		printf '%s\n' "$1" >>"$CALLS"
+		return 0
+	}
+	ss_now() { printf '%s\n' 1000; }
+	is_valid_integer() {
+		case "$1" in '' | *[!0-9]*) return 1 ;; esac
+		return 0
+	}
+
+	ss_status_reset
+	ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '6'
+	ss_spec_assert_eq "$(grep -c '^set_many$' "$CALLS")" '4'
+	ss_spec_assert_eq "$(grep -c '^reset$' "$CALLS")" '1'
+	ss_spec_assert_eq "$(grep -c '^clear_messages$' "$CALLS")" '1'
+
+	: >"$CALLS"
+	ss_status_mark_success 300
+	ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '2'
+	ss_spec_assert_file_line "$CALLS" 'set_many'
+
+	: >"$CALLS"
+	ss_status_mark_failure artifact_download_failed
+	ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '2'
+	ss_spec_assert_file_line "$CALLS" 'add_error'
 )

safeshield/spec/support/lifecycle_cases.sh

@@ -138,6 +138,7 @@ ss_case_refresh_state_machine() (
 			ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
 		else
 			! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
+			! grep -F 'dnsmasq_restart ' "$CALLS" >/dev/null
 		fi
 	}
 
@@ -157,7 +158,10 @@ ss_case_refresh_state_machine() (
 	ss_spec_assert_file_line "$CALLS" 'lock_close'
 	! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
 
-	assert_refresh_failure merge merge_failed 1
+	assert_refresh_failure merge merge_failed 0
+	assert_refresh_failure resolve artifact_resolve_failed 0
+	assert_refresh_failure download artifact_download_failed 0
+	assert_refresh_failure local_rules allowlist_build_failed 0
 	assert_refresh_failure install install_failed 1
 	assert_refresh_failure restart dnsmasq_restart_failed 1
 	assert_refresh_failure runtime dns_runtime_check_failed 1
@@ -313,7 +317,12 @@ ss_case_local_apply_state_machine() (
 			verify) code='local_blocklist_verification_failed' ;;
 		esac
 		ss_spec_assert_file_line "$CALLS" "local_failure $code"
-		ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
+		if [ "$scenario" = 'merge' ]; then
+			! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
+			! grep -Fx 'dnsmasq_restart' "$CALLS" >/dev/null
+		else
+			ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
+		fi
 		ss_spec_assert_file_line "$CALLS" 'lock_close'
 	done
 
@@ -333,54 +342,42 @@ ss_case_refresh_locking() (
 	set -eu
 	TMP="$(ss_spec_tmpdir)"
 	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
-	RUNNING_STATUS_FILE="$TMP/status.json"
 	SS_REFRESH_LOCK="$TMP/safeshield-refresh.lock"
-	SS_REFRESH_LOCK_FD=9
-	MOCK_FLOCK_DIR="$TMP/mock-flock.lock"
-
-	# Keep this regression test portable on development hosts such as macOS,
-	# where flock(1) is not installed by default. The production code still
-	# invokes flock -n/-u; this shim emulates those operations with atomic
-	# mkdir/rmdir so contention, release, timeout, and termination paths are
-	# exercised without depending on a host-specific flock binary.
-	flock() {
-		case "${1:-}" in
-			-n)
-				mkdir "$MOCK_FLOCK_DIR" 2>/dev/null
-				;;
-			-u)
-				rmdir "$MOCK_FLOCK_DIR" 2>/dev/null || true
-				return 0
-				;;
-			*)
-				return 2
-				;;
-		esac
-	}
-
-	# shellcheck disable=SC1091
+	# shellcheck disable=SC1090
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
+	# shellcheck disable=SC1090
 	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
 	is_valid_integer() { case "$1" in '' | *[!0-9]*) return 1 ;; *) return 0 ;; esac }
 	ss_should_stop() { return 1; }
 
-	mkdir "$MOCK_FLOCK_DIR"
+	ss_refresh_lock_open
+	ss_spec_assert_eq "$(cat "${SS_REFRESH_LOCK}.d/pid")" "$$"
+	# A separate shell must not acquire or release our live lock.
+	! sh -c '. "$1"; ss_lock_try "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "${SS_REFRESH_LOCK}.d"
+	ss_refresh_lock_close
+	ss_spec_assert_not_exists "${SS_REFRESH_LOCK}.d"
+
+	# Simulate an owner which crashed after writing pid.
+	mkdir "${SS_REFRESH_LOCK}.d"
+	printf '99999999\n' >"${SS_REFRESH_LOCK}.d/pid"
+	ss_refresh_lock_open
+	ss_refresh_lock_close
+
+	# A freshly-created empty lock must not be stolen while the owner writes pid.
+	mkdir "${SS_REFRESH_LOCK}.d"
 	! ss_refresh_lock_open
 	! ss_refresh_lock_open_wait 0
-	rmdir "$MOCK_FLOCK_DIR"
+	rmdir "${SS_REFRESH_LOCK}.d"
 
-	ss_refresh_lock_open
-	ss_spec_assert_exists "$MOCK_FLOCK_DIR"
-	ss_refresh_lock_close
-	ss_spec_assert_not_exists "$MOCK_FLOCK_DIR"
-
-	mkdir "$MOCK_FLOCK_DIR"
 	ss_should_stop() { return 0; }
+	mkdir "${SS_REFRESH_LOCK}.d"
+	printf '%s\n' "$$" >"${SS_REFRESH_LOCK}.d/pid"
 	set +e
 	ss_refresh_lock_open_wait 10
 	rc=$?
 	set -e
-	rmdir "$MOCK_FLOCK_DIR"
 	ss_spec_assert_eq "$rc" '130'
+	ss_refresh_lock_close
 )
 
 ss_case_service_lifecycle() (

safeshield/spec/support/locking_cases.sh

@@ -0,0 +1,114 @@
+#!/bin/sh
+# shellcheck shell=sh
+
+ss_case_lock_owner_and_contention() (
+	set -eu
+	TMP="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+	# shellcheck disable=SC1090
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
+	LOCK="$TMP/collector.lock"
+	ss_lock_try "$LOCK"
+	[ "$(cat "$LOCK/pid")" = "$$" ]
+	# Different process may not acquire the lock or remove its owner.
+	! sh -c '. "$1"; ss_lock_try "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "$LOCK"
+	! sh -c '. "$1"; ss_lock_release "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "$LOCK"
+	[ "$(cat "$LOCK/pid")" = "$$" ]
+	ss_lock_release "$LOCK"
+	[ ! -e "$LOCK" ]
+	# Stale pid may be reclaimed by exactly one contender.
+	mkdir "$LOCK"
+	printf '99999999\n' >"$LOCK/pid"
+	ss_lock_try "$LOCK"
+	[ "$(cat "$LOCK/pid")" = "$$" ]
+	ss_lock_release "$LOCK"
+	# Do not seize a newly created lock before its pid has been populated.
+	mkdir "$LOCK"
+	! ss_lock_try "$LOCK"
+	[ -d "$LOCK" ]
+	rmdir "$LOCK"
+	# An abandoned pid-less directory (including a crashed recovery guard)
+	# becomes reclaimable after its initialization grace period.
+	mkdir "$LOCK" "${LOCK}.reclaim"
+	touch -t 202001010000 "$LOCK" "${LOCK}.reclaim"
+	ss_lock_try "$LOCK"
+	ss_lock_release "$LOCK"
+)
+
+ss_case_lock_parallel() (
+	set -eu
+	TMP="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+	LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
+	LOCK="$TMP/write.lock"
+	RESULT="$TMP/results"
+	for id in 1 2 3 4 5 6 7 8; do
+		sh -c '
+			. "$1"
+			ss_lock_wait "$2" 15 || exit 1
+			printf "%s\n" "$3" >>"$4"
+			sleep 0.05
+			ss_lock_release "$2"
+		' sh "$LIB" "$LOCK" "$id" "$RESULT" &
+	done
+	wait
+	[ "$(wc -l <"$RESULT")" -eq 8 ]
+	[ ! -e "$LOCK" ]
+	[ ! -e "${LOCK}.reclaim" ]
+)
+
+ss_case_status_write_serialization() (
+	set -eu
+	TMP="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+	LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
+	STATUS="$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
+	# Test the real shell writer with a slow stand-in for the ucode serializer.
+	# A serializer must observe all previously committed updates under the same lock.
+	cat >"$TMP/writer.sh" <<'EOF_WRITER'
+#!/bin/sh
+. "$1"
+. "$2"
+RUNNING_STATUS_FILE="$3"
+RUNNING_STATUS_LOCK="$4"
+json_lock_open() { ss_lock_wait "$RUNNING_STATUS_LOCK" 10; }
+json_lock_close() { ss_lock_release "$RUNNING_STATUS_LOCK"; }
+ss_status_exec() {
+	key="$2"
+	[ -f "$RUNNING_STATUS_FILE" ] && cat "$RUNNING_STATUS_FILE"
+	sleep 0.05
+	printf '%s\n' "$key"
+}
+ss_status_apply set "$5" unused
+EOF_WRITER
+	for n in 1 2 3 4 5 6 7 8; do
+		sh "$TMP/writer.sh" "$LIB" "$STATUS" "$TMP/status.json" "$TMP/status.lock" "$n" &
+	done
+	wait
+	[ "$(wc -l <"$TMP/status.json")" -eq 8 ]
+	[ ! -e "$TMP/status.lock" ]
+	[ ! -e "$TMP/status.json.tmp" ]
+)
+
+ss_case_status_write_failure() (
+	set -eu
+	TMP="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+	# shellcheck disable=SC1090
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
+	# shellcheck disable=SC1090
+	. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
+	RUNNING_STATUS_LOCK="$TMP/status.lock"
+	RUNNING_STATUS_FILE="$TMP/status.json"
+	json_lock_open() { ss_lock_wait "$RUNNING_STATUS_LOCK" 10; }
+	json_lock_close() { ss_lock_release "$RUNNING_STATUS_LOCK"; }
+	printf 'original\n' >"$RUNNING_STATUS_FILE"
+	ss_status_exec() {
+		printf 'partial\n'
+		return 1
+	}
+	! ss_status_apply set key value
+	[ "$(cat "$RUNNING_STATUS_FILE")" = 'original' ]
+	[ ! -e "${RUNNING_STATUS_FILE}.tmp.$$" ]
+	[ ! -e "$RUNNING_STATUS_LOCK" ]
+)

safeshield/spec/support/runtime_cases.sh

@@ -115,6 +115,7 @@ EOF_IP
 	PATH="$TMP/bin:$PATH" \
 		SS_STATSD_FUNCTIONS_LIB="$TMP/functions.sh" \
 		SS_STATSD_CORE_LIB="$TMP/core.sh" \
+		SS_STATSD_LOCK_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" \
 		SS_STATSD_AWK_DIR="$SS_SPEC_ROOT/files/usr/lib/safeshield/statistics" \
 		SS_STATSD_POLL_COMMAND="$TMP/bin/stats-poll" \
 		SS_STATSD_GENERATION_ID='runtime-generation' \
@@ -580,6 +581,7 @@ EOF_CORE
 	write_payload "$SS_STATISTICS_UPLOAD_JSON_FILE" 7
 	upload_pending_create
 	ss_spec_assert_file_contains "$SS_STATISTICS_UPLOAD_PENDING_META_FILE" "$(printf 'recent\tgeneration-upload\t7')"
+	ss_should_terminate=0
 	MOCK_HTTP_MODE='stale'
 	export MOCK_HTTP_MODE
 	upload_send_pending

safeshield/spec/support/statistics_cases.sh

@@ -1272,3 +1272,37 @@ DATA
 	! grep -F '"bucket_start":1787950800' "$UPLOAD" >/dev/null
 	ss_spec_assert_eq "$(sed -n 's/.*"snapshot_seq":\([0-9][0-9]*\).*/\1/p' "$UPLOAD")" "$(sed -n 's/.*"snapshot_seq":\([0-9][0-9]*\).*/\1/p' "$JSON")"
 )
+
+# Verifies that caching the retention sweep never leaves expired history in
+# the local JSON, the persisted state, or the short cloud upload projection.
+ss_case_statistics_retention_cache() (
+	set -eu
+	TMP="$(ss_spec_tmpdir)"
+	trap 'rm -rf "$TMP"' EXIT HUP INT TERM
+	STATE="$TMP/state.tsv"
+	JSON="$TMP/statistics.json"
+	UPLOAD="$TMP/upload.json"
+	INPUT="$TMP/input.log"
+	LEASES="$TMP/dhcp.leases"
+	: >"$LEASES"
+	cat >"$INPUT" <<'DATA'
+daemon.info dnsmasq[1]: query[A] one.example from 192.168.1.1
+daemon.info dnsmasq[1]: query[A] two.example from 192.168.1.1
+daemon.info dnsmasq[1]: query[A] three.example from 192.168.1.1
+DATA
+	ss_statistics_awk \
+		-v state_file="$STATE" -v json_file="$JSON" \
+		-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
+		-v snapshot_interval=1 -v retention_hours=2 \
+		-v upload_window_hours=1 -v provisional_retention_hours=2 \
+		-v generation_seed='retention-test' \
+		-v fixed_now=1787950800 -v fixed_step=3600 <"$INPUT"
+
+	# The final event is in the third hour. The two-hour retention window
+	# contains exactly the second and third events.
+	ss_spec_assert_file_contains "$JSON" '"totals":{"queries":2,"blocked":0}'
+	ss_spec_assert_file_contains "$JSON" '"retention_hours":2'
+	ss_spec_assert_file_contains "$UPLOAD" '"totals":{"queries":1,"blocked":0}'
+	! grep -F 'bucket' "$STATE" | grep -F '1787950800' >/dev/null
+	ss_spec_assert_file_contains "$STATE" "$(printf 'bucket\t1787958000\t1\t0')"
+)

safeshield/spec/support/tooling_cases.sh

@@ -12,6 +12,8 @@ ss_case_tooling() (
 	[ -f "$SS_SPEC_ROOT/spec/statistics_spec.sh" ]
 	[ -f "$SS_SPEC_ROOT/spec/ucode_spec.sh" ]
 	[ -f "$SS_SPEC_ROOT/spec/tooling_spec.sh" ]
+	[ -f "$SS_SPEC_ROOT/spec/locking_spec.sh" ]
+	[ -f "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" ]
 	[ -f "$SS_SPEC_ROOT/scripts/lint.sh" ]
 	[ -f "$SS_SPEC_ROOT/.github/workflows/lint-shell.yml" ]
 	ss_spec_assert_file_contains "$SS_SPEC_ROOT/.pre-commit-config.yaml" 'entry: sh scripts/lint.sh'

safeshield/spec/support/ucode_cases.sh

@@ -48,6 +48,21 @@ ss_case_ucode() (
 	out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" set stage running)"
 	printf '%s\n' "$out" >"$status_file"
 	printf '%s\n' "$out" | grep -F '"stage":"running"' >/dev/null
+	# One invocation must update every pair without dropping existing status.
+	out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many status ready stage 'done' count 0 optional '')"
+	printf '%s\n' "$out" >"$status_file"
+	printf '%s\n' "$out" | grep -F '"status":"ready"' >/dev/null
+	printf '%s\n' "$out" | grep -F '"stage":"done"' >/dev/null
+	printf '%s\n' "$out" | grep -F '"count":"0"' >/dev/null
+	printf '%s\n' "$out" | grep -F '"optional":""' >/dev/null
+	printf '%s\n' "$out" | grep -F '"stage":"running"' >/dev/null && return 1
+	# Reject malformed pair lists, never serialize partial changes.
+	if "$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many incomplete >/dev/null 2>&1; then
+		return 1
+	fi
+	if "$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many '' value >/dev/null 2>&1; then
+		return 1
+	fi
 	out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" add_warning dnsmasq_warning)"
 	printf '%s\n' "$out" >"$status_file"
 	printf '%s\n' "$out" | grep -F '"code":"dnsmasq_warning"' >/dev/null