diff --git a/dockerd/Makefile b/dockerd/Makefile index 3b99365f5..0b60e2a78 100644 --- a/dockerd/Makefile +++ b/dockerd/Makefile @@ -2,7 +2,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=dockerd PKG_VERSION:=29.6.1 -PKG_RELEASE:=11 +PKG_RELEASE:=12 PKG_LICENSE:=Apache-2.0 PKG_LICENSE_FILES:=LICENSE PKG_CPE_ID:=cpe:/a:mobyproject:moby diff --git a/dockerd/files/dockerd.init b/dockerd/files/dockerd.init index 1ea3520cd..df7f8e158 100755 --- a/dockerd/files/dockerd.init +++ b/dockerd/files/dockerd.init @@ -215,6 +215,13 @@ start_service() { process_config + config_load 'dockerd' + config_get data_root globals data_root "/opt/docker/" + if df -h "${data_root%/}" | grep -qs "overlayfs:/overlay";then + mkdir -p /overlay${data_root%/} ${data_root%/} + mount --bind /overlay${data_root%/} ${data_root%/} + fi + procd_open_instance procd_set_param stderr 1 if [ -z "${DOCKERD_CONF}" ]; then diff --git a/safeshield/CHANGELOG.md b/safeshield/CHANGELOG.md index 5c3aabf22..1badbe358 100644 --- a/safeshield/CHANGELOG.md +++ b/safeshield/CHANGELOG.md @@ -1,5 +1,25 @@ # Changelog +## [0.3.27-r1] - 2026-11-11 + +- Bump version for release. + +## [0.3.26-r2] - 2026-11-11 + +### Fixed + +- Fixed an issue where an error in per-device statistics for a single historical hour could block the upload of all router protection statistics. +- Added safeguards to prevent incorrect per-device statistics from accumulating when DNS counters change unexpectedly. + +### Improved + +- Preserved router-wide statistics while excluding only the affected per-device details from uploads. The original stored statistics remain unchanged. +- Added the number of sanitized hourly buckets to the upload JSON for easier troubleshooting. + +### Tests + +- Added tests covering historical statistics errors, excessive per-device counter values, preservation of valid statistics, recovery after restarts, and anonymous device aggregation. + ## [0.3.26-r1] - 2026-11-02 - Bump version for release. diff --git a/safeshield/Makefile b/safeshield/Makefile index 981b7af1b..c306400ab 100644 --- a/safeshield/Makefile +++ b/safeshield/Makefile @@ -6,8 +6,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=safeshield -PKG_VERSION:=0.3.26 -PKG_RELEASE:=5 +PKG_VERSION:=0.3.27 +PKG_RELEASE:=6 PKG_MAINTAINER:=Beomjun Kang PKG_LICENSE:=GPL-3.0-or-later diff --git a/safeshield/README.md b/safeshield/README.md index d32353e32..ac3717ecf 100644 --- a/safeshield/README.md +++ b/safeshield/README.md @@ -487,6 +487,19 @@ consistent two-hour projection of the local hourly data. A full retained snapsho is sent on initial synchronization, after a failed upload has recovered, and periodically (approximately every 12 hours) to reconcile any missed window. +To avoid one malformed historical client hour blocking the entire retained +statistics upload, the collector checks per-client dnsmasq deltas against the +router-wide delta before recording them. A poll with inconsistent attribution +keeps the router counters but skips that poll's untrustworthy client breakdown. +When exporting JSON, any retained hour with client attribution exceeding the +router counters (or blocked counts exceeding queries) is projected with router +hourly totals intact and the affected hour's client attribution omitted. This +also repairs the cloud projection of previously stored inconsistent hours after +upgrade without deleting or rewriting the raw TSV/journal history. The optional +`source.sanitized_hours` field counts hours handled this way in each JSON +projection. Only the affected client-hour breakdown is unavailable; the other +hours and router totals remain intact. Schema v3 and upload ordering are unchanged. + The uploader preserves the exact pending JSON across network retries. If the Hub committed a request but the HTTP acknowledgement was lost, retrying the same `generation_id` and `snapshot_seq` is therefore idempotent. An HTTP 401 clears diff --git a/safeshield/files/etc/init.d/safeshield b/safeshield/files/etc/init.d/safeshield index 28050b9a6..e1fa55d46 100644 --- a/safeshield/files/etc/init.d/safeshield +++ b/safeshield/files/etc/init.d/safeshield @@ -16,7 +16,7 @@ IPKG_INSTROOT="${IPKG_INSTROOT:-}" export IPKG_INSTROOT readonly PKG_NAME='safeshield' -readonly PKG_VERSION='0.3.26-r1' +readonly PKG_VERSION='0.3.27-r1' if type extra_command >/dev/null 2>&1; then extra_command 'apply_local_rules' 'Apply local allow/block rules using the cached Hub artifact' diff --git a/safeshield/files/usr/lib/safeshield/statistics/50-output.awk b/safeshield/files/usr/lib/safeshield/statistics/50-output.awk index d64a0e0e5..78ed11a28 100644 --- a/safeshield/files/usr/lib/safeshield/statistics/50-output.awk +++ b/safeshield/files/usr/lib/safeshield/statistics/50-output.awk @@ -12,7 +12,60 @@ function statistics_first_hour(now, current_hour, cutoff, first_hour) { } function is_unknown_output_device(key) { - return (key == "unknown" || (device_mac[key] == "" && key ~ /^ip:/)) + return (key == "unknown" || key == "other" || (device_mac[key] == "" && key ~ /^ip:/)) +} + +# Never let one stale/corrupt per-client hour block a whole cloud reconciliation. +# This is a projection only: raw hourly counters and flash state are untouched. +# Suppress all client attribution for an inconsistent hour rather than guessing +# which client was wrong or inventing replacement counts. Router totals survive. +function prepare_safe_hourly(first_hour, last_hour, bucket, composite, parts, key, client_queries, client_blocked) { + sanitized_hour_count = 0 + for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { + delete masked_hour[bucket] + attributed_hour_queries[bucket] = 0 + attributed_hour_blocked[bucket] = 0 + safe_hour_blocked[bucket] = blocked[bucket] + 0 + if (safe_hour_blocked[bucket] > queries[bucket] + 0) { + safe_hour_blocked[bucket] = queries[bucket] + 0 + masked_hour[bucket] = 1 + } + } + + for (composite in device_hour_queries) { + split(composite, parts, SUBSEP) + key = parts[1] + bucket = parts[2] + 0 + if (!(key in device_seen) || bucket < first_hour || bucket > last_hour) { + continue + } + client_queries = device_hour_queries[composite] + 0 + client_blocked = device_hour_blocked[composite] + 0 + attributed_hour_queries[bucket] += client_queries + attributed_hour_blocked[bucket] += client_blocked + if (client_blocked > client_queries) { + masked_hour[bucket] = 1 + } + } + + for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { + if (attributed_hour_queries[bucket] > queries[bucket] + 0 || \ + attributed_hour_blocked[bucket] > safe_hour_blocked[bucket]) { + masked_hour[bucket] = 1 + } + if (masked_hour[bucket]) { + sanitized_hour_count++ + } + } +} + +function clear_safe_hourly(first_hour, last_hour, bucket) { + for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { + delete masked_hour[bucket] + delete attributed_hour_queries[bucket] + delete attributed_hour_blocked[bucket] + delete safe_hour_blocked[bucket] + } } function save_json_range(path, now, first_hour, last_hour, include_empty_devices, tmp, bucket, comma, key, identified, device_comma, identity_comma, composite, device_first_hour, device_last_hour, persistence_enabled, persistent, healthy, volatile_state, storage, persistence_mode, truncated, range_queries, range_blocked, device_range_queries, device_range_blocked, device_has_rows, unknown_has_rows, unknown_queries, unknown_blocked) { @@ -32,12 +85,18 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices storage = (persistence_enabled == "true") ? "tmpfs+flash" : "tmpfs" persistence_mode = (persistent_journal_file != "") ? "journal" : ((persistent_state_file != "") ? "snapshot" : "none") truncated = devices_truncated ? "true" : "false" + # Keep malformed source metadata from invalidating an otherwise sound upload. + safe_untracked_blocked = source_untracked_blocked + if (safe_untracked_blocked > source_untracked_queries) { + safe_untracked_blocked = source_untracked_queries + } + prepare_safe_hourly(first_hour, last_hour) range_queries = 0 range_blocked = 0 for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { range_queries += queries[bucket] + 0 - range_blocked += blocked[bucket] + 0 + range_blocked += safe_hour_blocked[bucket] delete unknown_hour_queries[bucket] delete unknown_hour_blocked[bucket] } @@ -60,6 +119,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices continue } for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { + if (masked_hour[bucket]) { + continue + } composite = device_bucket_key(key, bucket) if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) { continue @@ -87,9 +149,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices json_escape(source_instance_id), json_escape(source_transport_scope) >> tmp printf "\"client_capacity\":%d,\"tracked_clients\":%d,\"untracked_queries\":%d,", \ source_client_capacity, source_tracked_clients, source_untracked_queries >> tmp - printf "\"untracked_blocked\":%d,", source_untracked_blocked >> tmp - printf "\"poll_error_count\":%d,\"last_error_at\":%d},", \ - source_error_count, source_last_error_at >> tmp + printf "\"untracked_blocked\":%d,", safe_untracked_blocked >> tmp + printf "\"poll_error_count\":%d,\"last_error_at\":%d,\"sanitized_hours\":%d},", \ + source_error_count, source_last_error_at, sanitized_hour_count >> tmp printf "\"started_at\":%d,\"session_started_at\":%d,\"updated_at\":%d,", started_at, session_started_at, now >> tmp printf "\"retention_hours\":%d,", retention_hours >> tmp printf "\"device_limit\":%d,\"devices_truncated\":%s,", max_devices, truncated >> tmp @@ -98,7 +160,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices comma = "" for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { - printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", comma, bucket, queries[bucket] + 0, blocked[bucket] + 0 >> tmp + printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", comma, bucket, queries[bucket] + 0, safe_hour_blocked[bucket] >> tmp comma = "," } printf "],\"devices\":[" >> tmp @@ -122,6 +184,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices device_has_rows = 0 if (device_first_hour <= device_last_hour) { for (bucket = device_first_hour; bucket <= device_last_hour; bucket += 3600) { + if (masked_hour[bucket]) { + continue + } composite = device_bucket_key(key, bucket) if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) { continue @@ -166,6 +231,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices device_comma = "" if (device_first_hour <= device_last_hour) { for (bucket = device_first_hour; bucket <= device_last_hour; bucket += 3600) { + if (masked_hour[bucket]) { + continue + } composite = device_bucket_key(key, bucket) if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) { continue @@ -184,7 +252,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices comma, unknown_queries, unknown_blocked >> tmp device_comma = "" for (bucket = first_hour; bucket <= last_hour; bucket += 3600) { - if (!((bucket in unknown_hour_queries) || (bucket in unknown_hour_blocked))) { + if (masked_hour[bucket] || !((bucket in unknown_hour_queries) || (bucket in unknown_hour_blocked))) { continue } printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", \ @@ -201,6 +269,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices delete unknown_hour_queries[bucket] delete unknown_hour_blocked[bucket] } + clear_safe_hourly(first_hour, last_hour) return replace_file(tmp, path) } diff --git a/safeshield/files/usr/lib/safeshield/statistics/90-main.awk b/safeshield/files/usr/lib/safeshield/statistics/90-main.awk index 1f50167be..541e51baa 100644 --- a/safeshield/files/usr/lib/safeshield/statistics/90-main.awk +++ b/safeshield/files/usr/lib/safeshield/statistics/90-main.awk @@ -57,7 +57,7 @@ function replace_source_clients( address) { } } -function apply_source_snapshot( now, bucket, had_source, same_epoch, query_delta, blocked_delta, internal_queries, internal_blocked, address, client_queries_delta, client_blocked_delta) { +function apply_source_snapshot( now, bucket, had_source, same_epoch, query_delta, blocked_delta, internal_queries, internal_blocked, address, client_queries_delta, client_blocked_delta, attributed_queries, attributed_blocked, invalid_attribution) { if (!poll_active || poll_instance_id == "") { return } @@ -78,6 +78,12 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de internal_queries = 0 internal_blocked = 0 + attributed_queries = 0 + attributed_blocked = 0 + invalid_attribution = 0 + # Calculate every delta before recording any client. dnsmasq client counters + # can reappear with an old baseline even as global counters advance only + # slightly. Such an inconsistent poll must not poison the current hour. for (address in poll_client_seen) { if (!had_source) { client_queries_delta = 0 @@ -89,15 +95,18 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de client_blocked_delta = source_counter_delta(poll_client_blocked[address], \ source_client_blocked[address], same_epoch) } - + poll_client_query_delta[address] = client_queries_delta + poll_client_block_delta[address] = client_blocked_delta if (is_internal_statistics_client(address)) { internal_queries += client_queries_delta internal_blocked += client_blocked_delta - continue } - - if (client_queries_delta > 0 || client_blocked_delta > 0) { - record_device(address, client_queries_delta, client_blocked_delta, now) + else { + attributed_queries += client_queries_delta + attributed_blocked += client_blocked_delta + if (client_blocked_delta > client_queries_delta) { + invalid_attribution = 1 + } } } @@ -109,6 +118,30 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de if (blocked_delta < 0) { blocked_delta = 0 } + if (blocked_delta > query_delta) { + # Do not manufacture blocked queries from inconsistent source deltas. + blocked_delta = query_delta + invalid_attribution = 1 + } + if (attributed_queries > query_delta || attributed_blocked > blocked_delta) { + invalid_attribution = 1 + } + if (!invalid_attribution) { + for (address in poll_client_seen) { + if (is_internal_statistics_client(address)) { + continue + } + client_queries_delta = poll_client_query_delta[address] + client_blocked_delta = poll_client_block_delta[address] + if (client_queries_delta > 0 || client_blocked_delta > 0) { + record_device(address, client_queries_delta, client_blocked_delta, now) + } + } + } + for (address in poll_client_seen) { + delete poll_client_query_delta[address] + delete poll_client_block_delta[address] + } if (query_delta > 0 || blocked_delta > 0) { aggregate_dirty = 1 diff --git a/safeshield/spec/statistics_spec.sh b/safeshield/spec/statistics_spec.sh index 44e6d6194..eccd76fb8 100755 --- a/safeshield/spec/statistics_spec.sh +++ b/safeshield/spec/statistics_spec.sh @@ -119,6 +119,34 @@ Describe 'SafeShield statistics behavior' The error should equal '' End + It '잘못된 과거 통계 한 시간이 전체 업로드와 재시작 복구를 막지 않는다' + When call ss_case_statistics_historical_inconsistency_recovery + The status should be success + The output should equal '' + The error should equal '' + End + + It 'dnsmasq 카운터가 과대 집계되면 해당 회차만 격리하고 다음 수집을 재개한다' + When call ss_case_statistics_source_delta_overrun_guard + The status should be success + The output should equal '' + The error should equal '' + End + + It '여러 기기의 합계 및 차단 수 이상이 정상 시간대를 오염시키지 않는다' + When call ss_case_statistics_aggregate_hour_guard + The status should be success + The output should equal '' + The error should equal '' + End + + It '기기 수 초과 집계가 잘못된 장치 식별자로 업로드되지 않는다' + When call ss_case_statistics_overflow_identity_projection + The status should be success + The output should equal '' + The error should equal '' + End + It 'prunes expired statistics while preserving the short upload window' When call ss_case_statistics_retention_cache The status should be success diff --git a/safeshield/spec/support/statistics_cases.sh b/safeshield/spec/support/statistics_cases.sh index 74ece4215..1e12db521 100755 --- a/safeshield/spec/support/statistics_cases.sh +++ b/safeshield/spec/support/statistics_cases.sh @@ -1306,3 +1306,148 @@ DATA ! grep -F 'bucket' "$STATE" | grep -F '1787950800' >/dev/null ss_spec_assert_file_contains "$STATE" "$(printf 'bucket\t1787958000\t1\t0')" ) + +# A corrupt historical client bucket must never hold newer valid router data +# hostage during a full cloud upload. The raw state stays available to debug. +ss_case_statistics_historical_inconsistency_recovery() ( + set -eu + TMP="$(ss_spec_tmpdir)" + trap 'rm -rf "$TMP"' EXIT HUP INT TERM + STATE="$TMP/state.tsv" + JSON="$TMP/statistics.json" + UPLOAD="$TMP/upload.json" + LEASES="$TMP/leases" + : >"$LEASES" + cat >"$STATE" <<'STATE' +meta 1794034800 1794389700 10 2 0 0 6 1794389700 1 0 0 0 0 generation-recovery 135290 135290 +bucket 1794034800 0 0 +bucket 1794387600 10 2 +device aa:bb:cc:dd:ee:ff aa:bb:cc:dd:ee:ff 192.168.1.20 phone 10 3 * * +device_bucket aa:bb:cc:dd:ee:ff 1794034800 6 2 +device_bucket aa:bb:cc:dd:ee:ff 1794387600 4 1 +device ip:192.168.1.30 * 192.168.1.30 * 3 1 * * +device_bucket ip:192.168.1.30 1794387600 3 1 +STATE + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \ + -v snapshot_interval=60 -v retention_hours=168 \ + -v upload_window_hours=2 -v fixed_now=1794389810 /dev/null + # Restarting while the same bad flash-backed hour remains still produces + # a valid full snapshot. No manual reset or 7-day wait is necessary. + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \ + -v snapshot_interval=60 -v retention_hours=168 \ + -v upload_window_hours=2 -v fixed_now=1794389870 "$LEASES" + cat >"$INPUT" <<'DATA' +snapshot one-epoch udp 128 1 0 0 10 2 +client 192.168.1.20 4 1 +commit +DATA + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \ + -v fixed_now=1787950800 <"$INPUT" + cat >"$INPUT" <<'DATA' +snapshot one-epoch udp 128 1 0 0 20 5 +client 192.168.1.20 22 7 +commit +DATA + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \ + -v fixed_now=1787950860 <"$INPUT" + ss_spec_assert_file_contains "$JSON" '"totals":{"queries":10,"blocked":3}' + ss_spec_assert_file_contains "$JSON" '"sanitized_hours":0' + ! grep -F '"queries":18,"blocked":6' "$STATE" >/dev/null + cat >"$INPUT" <<'DATA' +snapshot one-epoch udp 128 1 0 0 27 7 +client 192.168.1.20 26 8 +commit +DATA + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \ + -v fixed_now=1787950920 <"$INPUT" + ss_spec_assert_file_contains "$JSON" '"totals":{"queries":17,"blocked":5}' + ss_spec_assert_file_contains "$JSON" '"id":"aa:bb:cc:dd:ee:ff","mac":"aa:bb:cc:dd:ee:ff","ip":"192.168.1.20","hostname":"phone","identified":true,"queries":4,"blocked":1' + ss_spec_assert_file_contains "$JSON" '"sanitized_hours":0' +) + +# Two individually valid clients can jointly exceed the router count, and an +# invalid blocked total must not break the JSON schema or poison other hours. +ss_case_statistics_aggregate_hour_guard() ( + set -eu + TMP="$(ss_spec_tmpdir)" + trap 'rm -rf "$TMP"' EXIT HUP INT TERM + STATE="$TMP/state.tsv" + JSON="$TMP/statistics.json" + LEASES="$TMP/leases" + : >"$LEASES" + cat >"$STATE" <<'STATE' +meta 1787950800 1787954400 7 9 0 0 6 1787954400 +bucket 1787950800 2 5 +bucket 1787954400 5 4 +device aa:bb:cc:dd:ee:ff aa:bb:cc:dd:ee:ff 192.168.1.20 phone 4 2 * * +device_bucket aa:bb:cc:dd:ee:ff 1787950800 2 1 +device_bucket aa:bb:cc:dd:ee:ff 1787954400 2 1 +device 11:22:33:44:55:66 11:22:33:44:55:66 192.168.1.30 laptop 5 2 * * +device_bucket 11:22:33:44:55:66 1787950800 1 1 +device_bucket 11:22:33:44:55:66 1787954400 4 1 +STATE + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v lease_file="$LEASES" -v fixed_now=1787954400 /dev/null +) + +# Capacity overflow is an anonymous aggregate, not an invalid non-MAC client. +ss_case_statistics_overflow_identity_projection() ( + set -eu + TMP="$(ss_spec_tmpdir)" + trap 'rm -rf "$TMP"' EXIT HUP INT TERM + STATE="$TMP/state.tsv" + JSON="$TMP/statistics.json" + LEASES="$TMP/leases" + : >"$LEASES" + cat >"$STATE" <<'STATE' +meta 1787950800 1787950800 10 3 0 0 6 1787950800 +device other * * Other devices 4 1 * * +device_bucket other 1787950800 4 1 +bucket 1787950800 10 3 +source epoch udp 128 1 0 10 3 1 0 0 5 +STATE + ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \ + -v lease_file="$LEASES" -v fixed_now=1787950800