🍕 Sync 2026-11-11 20:57:55

This commit is contained in:
github-actions[bot] committed 2026-11-11 20:57:55 +08:00
1 parent 8fc43cea4d
commit 86de425f45
10 files changed
+332 -17

No files matched your search

+20
View File
@@ -1,5 +1,25 @@
# Changelog
## [0.3.27-r1] - 2026-11-11
- Bump version for release.
## [0.3.26-r2] - 2026-11-11
### Fixed
- Fixed an issue where an error in per-device statistics for a single historical hour could block the upload of all router protection statistics.
- Added safeguards to prevent incorrect per-device statistics from accumulating when DNS counters change unexpectedly.
### Improved
- Preserved router-wide statistics while excluding only the affected per-device details from uploads. The original stored statistics remain unchanged.
- Added the number of sanitized hourly buckets to the upload JSON for easier troubleshooting.
### Tests
- Added tests covering historical statistics errors, excessive per-device counter values, preservation of valid statistics, recovery after restarts, and anonymous device aggregation.
## [0.3.26-r1] - 2026-11-02
- Bump version for release.
+2 -2
View File
@@ -6,8 +6,8 @@
include $(TOPDIR)/rules.mk
PKG_NAME:=safeshield
PKG_VERSION:=0.3.26
PKG_RELEASE:=5
PKG_VERSION:=0.3.27
PKG_RELEASE:=6
PKG_MAINTAINER:=Beomjun Kang <kals323@gmail.com>
PKG_LICENSE:=GPL-3.0-or-later
+13
View File
@@ -487,6 +487,19 @@ consistent two-hour projection of the local hourly data. A full retained snapsho
is sent on initial synchronization, after a failed upload has recovered, and
periodically (approximately every 12 hours) to reconcile any missed window.
To avoid one malformed historical client hour blocking the entire retained
statistics upload, the collector checks per-client dnsmasq deltas against the
router-wide delta before recording them. A poll with inconsistent attribution
keeps the router counters but skips that poll's untrustworthy client breakdown.
When exporting JSON, any retained hour with client attribution exceeding the
router counters (or blocked counts exceeding queries) is projected with router
hourly totals intact and the affected hour's client attribution omitted. This
also repairs the cloud projection of previously stored inconsistent hours after
upgrade without deleting or rewriting the raw TSV/journal history. The optional
`source.sanitized_hours` field counts hours handled this way in each JSON
projection. Only the affected client-hour breakdown is unavailable; the other
hours and router totals remain intact. Schema v3 and upload ordering are unchanged.
The uploader preserves the exact pending JSON across network retries. If the Hub
committed a request but the HTTP acknowledgement was lost, retrying the same
`generation_id` and `snapshot_seq` is therefore idempotent. An HTTP 401 clears
+1 -1
View File
@@ -16,7 +16,7 @@ IPKG_INSTROOT="${IPKG_INSTROOT:-}"
export IPKG_INSTROOT
readonly PKG_NAME='safeshield'
readonly PKG_VERSION='0.3.26-r1'
readonly PKG_VERSION='0.3.27-r1'
if type extra_command >/dev/null 2>&1; then
extra_command 'apply_local_rules' 'Apply local allow/block rules using the cached Hub artifact'
@@ -12,7 +12,60 @@ function statistics_first_hour(now, current_hour, cutoff, first_hour) {
}
function is_unknown_output_device(key) {
return (key == "unknown" || (device_mac[key] == "" && key ~ /^ip:/))
return (key == "unknown" || key == "other" || (device_mac[key] == "" && key ~ /^ip:/))
}
# Never let one stale/corrupt per-client hour block a whole cloud reconciliation.
# This is a projection only: raw hourly counters and flash state are untouched.
# Suppress all client attribution for an inconsistent hour rather than guessing
# which client was wrong or inventing replacement counts. Router totals survive.
function prepare_safe_hourly(first_hour, last_hour, bucket, composite, parts, key, client_queries, client_blocked) {
sanitized_hour_count = 0
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
delete masked_hour[bucket]
attributed_hour_queries[bucket] = 0
attributed_hour_blocked[bucket] = 0
safe_hour_blocked[bucket] = blocked[bucket] + 0
if (safe_hour_blocked[bucket] > queries[bucket] + 0) {
safe_hour_blocked[bucket] = queries[bucket] + 0
masked_hour[bucket] = 1
}
}
for (composite in device_hour_queries) {
split(composite, parts, SUBSEP)
key = parts[1]
bucket = parts[2] + 0
if (!(key in device_seen) || bucket < first_hour || bucket > last_hour) {
continue
}
client_queries = device_hour_queries[composite] + 0
client_blocked = device_hour_blocked[composite] + 0
attributed_hour_queries[bucket] += client_queries
attributed_hour_blocked[bucket] += client_blocked
if (client_blocked > client_queries) {
masked_hour[bucket] = 1
}
}
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
if (attributed_hour_queries[bucket] > queries[bucket] + 0 || \
attributed_hour_blocked[bucket] > safe_hour_blocked[bucket]) {
masked_hour[bucket] = 1
}
if (masked_hour[bucket]) {
sanitized_hour_count++
}
}
}
function clear_safe_hourly(first_hour, last_hour, bucket) {
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
delete masked_hour[bucket]
delete attributed_hour_queries[bucket]
delete attributed_hour_blocked[bucket]
delete safe_hour_blocked[bucket]
}
}
function save_json_range(path, now, first_hour, last_hour, include_empty_devices, tmp, bucket, comma, key, identified, device_comma, identity_comma, composite, device_first_hour, device_last_hour, persistence_enabled, persistent, healthy, volatile_state, storage, persistence_mode, truncated, range_queries, range_blocked, device_range_queries, device_range_blocked, device_has_rows, unknown_has_rows, unknown_queries, unknown_blocked) {
@@ -32,12 +85,18 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
storage = (persistence_enabled == "true") ? "tmpfs+flash" : "tmpfs"
persistence_mode = (persistent_journal_file != "") ? "journal" : ((persistent_state_file != "") ? "snapshot" : "none")
truncated = devices_truncated ? "true" : "false"
# Keep malformed source metadata from invalidating an otherwise sound upload.
safe_untracked_blocked = source_untracked_blocked
if (safe_untracked_blocked > source_untracked_queries) {
safe_untracked_blocked = source_untracked_queries
}
prepare_safe_hourly(first_hour, last_hour)
range_queries = 0
range_blocked = 0
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
range_queries += queries[bucket] + 0
range_blocked += blocked[bucket] + 0
range_blocked += safe_hour_blocked[bucket]
delete unknown_hour_queries[bucket]
delete unknown_hour_blocked[bucket]
}
@@ -60,6 +119,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
continue
}
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
if (masked_hour[bucket]) {
continue
}
composite = device_bucket_key(key, bucket)
if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) {
continue
@@ -87,9 +149,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
json_escape(source_instance_id), json_escape(source_transport_scope) >> tmp
printf "\"client_capacity\":%d,\"tracked_clients\":%d,\"untracked_queries\":%d,", \
source_client_capacity, source_tracked_clients, source_untracked_queries >> tmp
printf "\"untracked_blocked\":%d,", source_untracked_blocked >> tmp
printf "\"poll_error_count\":%d,\"last_error_at\":%d},", \
source_error_count, source_last_error_at >> tmp
printf "\"untracked_blocked\":%d,", safe_untracked_blocked >> tmp
printf "\"poll_error_count\":%d,\"last_error_at\":%d,\"sanitized_hours\":%d},", \
source_error_count, source_last_error_at, sanitized_hour_count >> tmp
printf "\"started_at\":%d,\"session_started_at\":%d,\"updated_at\":%d,", started_at, session_started_at, now >> tmp
printf "\"retention_hours\":%d,", retention_hours >> tmp
printf "\"device_limit\":%d,\"devices_truncated\":%s,", max_devices, truncated >> tmp
@@ -98,7 +160,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
comma = ""
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", comma, bucket, queries[bucket] + 0, blocked[bucket] + 0 >> tmp
printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", comma, bucket, queries[bucket] + 0, safe_hour_blocked[bucket] >> tmp
comma = ","
}
printf "],\"devices\":[" >> tmp
@@ -122,6 +184,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
device_has_rows = 0
if (device_first_hour <= device_last_hour) {
for (bucket = device_first_hour; bucket <= device_last_hour; bucket += 3600) {
if (masked_hour[bucket]) {
continue
}
composite = device_bucket_key(key, bucket)
if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) {
continue
@@ -166,6 +231,9 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
device_comma = ""
if (device_first_hour <= device_last_hour) {
for (bucket = device_first_hour; bucket <= device_last_hour; bucket += 3600) {
if (masked_hour[bucket]) {
continue
}
composite = device_bucket_key(key, bucket)
if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) {
continue
@@ -184,7 +252,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
comma, unknown_queries, unknown_blocked >> tmp
device_comma = ""
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
if (!((bucket in unknown_hour_queries) || (bucket in unknown_hour_blocked))) {
if (masked_hour[bucket] || !((bucket in unknown_hour_queries) || (bucket in unknown_hour_blocked))) {
continue
}
printf "%s{\"bucket_start\":%d,\"queries\":%d,\"blocked\":%d}", \
@@ -201,6 +269,7 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
delete unknown_hour_queries[bucket]
delete unknown_hour_blocked[bucket]
}
clear_safe_hourly(first_hour, last_hour)
return replace_file(tmp, path)
}
@@ -57,7 +57,7 @@ function replace_source_clients( address) {
}
}
function apply_source_snapshot( now, bucket, had_source, same_epoch, query_delta, blocked_delta, internal_queries, internal_blocked, address, client_queries_delta, client_blocked_delta) {
function apply_source_snapshot( now, bucket, had_source, same_epoch, query_delta, blocked_delta, internal_queries, internal_blocked, address, client_queries_delta, client_blocked_delta, attributed_queries, attributed_blocked, invalid_attribution) {
if (!poll_active || poll_instance_id == "") {
return
}
@@ -78,6 +78,12 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de
internal_queries = 0
internal_blocked = 0
attributed_queries = 0
attributed_blocked = 0
invalid_attribution = 0
# Calculate every delta before recording any client. dnsmasq client counters
# can reappear with an old baseline even as global counters advance only
# slightly. Such an inconsistent poll must not poison the current hour.
for (address in poll_client_seen) {
if (!had_source) {
client_queries_delta = 0
@@ -89,15 +95,18 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de
client_blocked_delta = source_counter_delta(poll_client_blocked[address], \
source_client_blocked[address], same_epoch)
}
poll_client_query_delta[address] = client_queries_delta
poll_client_block_delta[address] = client_blocked_delta
if (is_internal_statistics_client(address)) {
internal_queries += client_queries_delta
internal_blocked += client_blocked_delta
continue
}
if (client_queries_delta > 0 || client_blocked_delta > 0) {
record_device(address, client_queries_delta, client_blocked_delta, now)
else {
attributed_queries += client_queries_delta
attributed_blocked += client_blocked_delta
if (client_blocked_delta > client_queries_delta) {
invalid_attribution = 1
}
}
}
@@ -109,6 +118,30 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de
if (blocked_delta < 0) {
blocked_delta = 0
}
if (blocked_delta > query_delta) {
# Do not manufacture blocked queries from inconsistent source deltas.
blocked_delta = query_delta
invalid_attribution = 1
}
if (attributed_queries > query_delta || attributed_blocked > blocked_delta) {
invalid_attribution = 1
}
if (!invalid_attribution) {
for (address in poll_client_seen) {
if (is_internal_statistics_client(address)) {
continue
}
client_queries_delta = poll_client_query_delta[address]
client_blocked_delta = poll_client_block_delta[address]
if (client_queries_delta > 0 || client_blocked_delta > 0) {
record_device(address, client_queries_delta, client_blocked_delta, now)
}
}
}
for (address in poll_client_seen) {
delete poll_client_query_delta[address]
delete poll_client_block_delta[address]
}
if (query_delta > 0 || blocked_delta > 0) {
aggregate_dirty = 1
+28
View File
@@ -119,6 +119,34 @@ Describe 'SafeShield statistics behavior'
The error should equal ''
End
It '잘못된 과거 통계 한 시간이 전체 업로드와 재시작 복구를 막지 않는다'
When call ss_case_statistics_historical_inconsistency_recovery
The status should be success
The output should equal ''
The error should equal ''
End
It 'dnsmasq 카운터가 과대 집계되면 해당 회차만 격리하고 다음 수집을 재개한다'
When call ss_case_statistics_source_delta_overrun_guard
The status should be success
The output should equal ''
The error should equal ''
End
It '여러 기기의 합계 및 차단 수 이상이 정상 시간대를 오염시키지 않는다'
When call ss_case_statistics_aggregate_hour_guard
The status should be success
The output should equal ''
The error should equal ''
End
It '기기 수 초과 집계가 잘못된 장치 식별자로 업로드되지 않는다'
When call ss_case_statistics_overflow_identity_projection
The status should be success
The output should equal ''
The error should equal ''
End
It 'prunes expired statistics while preserving the short upload window'
When call ss_case_statistics_retention_cache
The status should be success
+145
View File
@@ -1306,3 +1306,148 @@ DATA
! grep -F 'bucket' "$STATE" | grep -F '1787950800' >/dev/null
ss_spec_assert_file_contains "$STATE" "$(printf 'bucket\t1787958000\t1\t0')"
)
# A corrupt historical client bucket must never hold newer valid router data
# hostage during a full cloud upload. The raw state stays available to debug.
ss_case_statistics_historical_inconsistency_recovery() (
set -eu
TMP="$(ss_spec_tmpdir)"
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
STATE="$TMP/state.tsv"
JSON="$TMP/statistics.json"
UPLOAD="$TMP/upload.json"
LEASES="$TMP/leases"
: >"$LEASES"
cat >"$STATE" <<'STATE'
meta 1794034800 1794389700 10 2 0 0 6 1794389700 1 0 0 0 0 generation-recovery 135290 135290
bucket 1794034800 0 0
bucket 1794387600 10 2
device aa:bb:cc:dd:ee:ff aa:bb:cc:dd:ee:ff 192.168.1.20 phone 10 3 * *
device_bucket aa:bb:cc:dd:ee:ff 1794034800 6 2
device_bucket aa:bb:cc:dd:ee:ff 1794387600 4 1
device ip:192.168.1.30 * 192.168.1.30 * 3 1 * *
device_bucket ip:192.168.1.30 1794387600 3 1
STATE
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
-v snapshot_interval=60 -v retention_hours=168 \
-v upload_window_hours=2 -v fixed_now=1794389810 </dev/null
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":10,"blocked":2}'
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":1'
ss_spec_assert_file_contains "$UPLOAD" '"sanitized_hours":0'
ss_spec_assert_file_contains "$JSON" '"bucket_start":1794034800,"queries":0,"blocked":0'
ss_spec_assert_file_contains "$JSON" '"id":"aa:bb:cc:dd:ee:ff","mac":"aa:bb:cc:dd:ee:ff","ip":"192.168.1.20","hostname":"phone","identified":true,"queries":4,"blocked":1'
ss_spec_assert_file_contains "$JSON" '"id":"unknown","mac":"","ip":"","hostname":"Unknown devices","identified":false,"queries":3,"blocked":1'
ss_spec_assert_file_contains "$UPLOAD" '"totals":{"queries":10,"blocked":2}'
ss_spec_assert_file_contains "$STATE" "$(printf 'device_bucket\taa:bb:cc:dd:ee:ff\t1794034800\t6\t2')"
! grep -F '"bucket_start":1794034800,"queries":6' "$JSON" >/dev/null
# Restarting while the same bad flash-backed hour remains still produces
# a valid full snapshot. No manual reset or 7-day wait is necessary.
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
-v snapshot_interval=60 -v retention_hours=168 \
-v upload_window_hours=2 -v fixed_now=1794389870 </dev/null
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":1'
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":10,"blocked":2}'
)
# A source client can return with an older counter baseline than the global
# dnsmasq epoch. Skip that poll's attribution and resume on the next good poll.
ss_case_statistics_source_delta_overrun_guard() (
set -eu
TMP="$(ss_spec_tmpdir)"
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
STATE="$TMP/state.tsv"
JSON="$TMP/statistics.json"
UPLOAD="$TMP/upload.json"
LEASES="$TMP/leases"
INPUT="$TMP/poll.tsv"
printf '%s\n' '1789000000 aa:bb:cc:dd:ee:ff 192.168.1.20 phone *' >"$LEASES"
cat >"$INPUT" <<'DATA'
snapshot one-epoch udp 128 1 0 0 10 2
client 192.168.1.20 4 1
commit
DATA
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
-v fixed_now=1787950800 <"$INPUT"
cat >"$INPUT" <<'DATA'
snapshot one-epoch udp 128 1 0 0 20 5
client 192.168.1.20 22 7
commit
DATA
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
-v fixed_now=1787950860 <"$INPUT"
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":10,"blocked":3}'
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":0'
! grep -F '"queries":18,"blocked":6' "$STATE" >/dev/null
cat >"$INPUT" <<'DATA'
snapshot one-epoch udp 128 1 0 0 27 7
client 192.168.1.20 26 8
commit
DATA
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
-v fixed_now=1787950920 <"$INPUT"
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":17,"blocked":5}'
ss_spec_assert_file_contains "$JSON" '"id":"aa:bb:cc:dd:ee:ff","mac":"aa:bb:cc:dd:ee:ff","ip":"192.168.1.20","hostname":"phone","identified":true,"queries":4,"blocked":1'
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":0'
)
# Two individually valid clients can jointly exceed the router count, and an
# invalid blocked total must not break the JSON schema or poison other hours.
ss_case_statistics_aggregate_hour_guard() (
set -eu
TMP="$(ss_spec_tmpdir)"
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
STATE="$TMP/state.tsv"
JSON="$TMP/statistics.json"
LEASES="$TMP/leases"
: >"$LEASES"
cat >"$STATE" <<'STATE'
meta 1787950800 1787954400 7 9 0 0 6 1787954400
bucket 1787950800 2 5
bucket 1787954400 5 4
device aa:bb:cc:dd:ee:ff aa:bb:cc:dd:ee:ff 192.168.1.20 phone 4 2 * *
device_bucket aa:bb:cc:dd:ee:ff 1787950800 2 1
device_bucket aa:bb:cc:dd:ee:ff 1787954400 2 1
device 11:22:33:44:55:66 11:22:33:44:55:66 192.168.1.30 laptop 5 2 * *
device_bucket 11:22:33:44:55:66 1787950800 1 1
device_bucket 11:22:33:44:55:66 1787954400 4 1
STATE
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v lease_file="$LEASES" -v fixed_now=1787954400 </dev/null
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":7,"blocked":6}'
ss_spec_assert_file_contains "$JSON" '"bucket_start":1787950800,"queries":2,"blocked":2'
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":2'
# The second hour has six client queries but only five router queries.
# Both affected hours must be isolated without changing router counters.
! grep -F '"bucket_start":1787950800,"queries":2,"blocked":1' "$JSON" >/dev/null
)
# Capacity overflow is an anonymous aggregate, not an invalid non-MAC client.
ss_case_statistics_overflow_identity_projection() (
set -eu
TMP="$(ss_spec_tmpdir)"
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
STATE="$TMP/state.tsv"
JSON="$TMP/statistics.json"
LEASES="$TMP/leases"
: >"$LEASES"
cat >"$STATE" <<'STATE'
meta 1787950800 1787950800 10 3 0 0 6 1787950800
device other * * Other devices 4 1 * *
device_bucket other 1787950800 4 1
bucket 1787950800 10 3
source epoch udp 128 1 0 10 3 1 0 0 5
STATE
ss_statistics_awk -v state_file="$STATE" -v json_file="$JSON" \
-v lease_file="$LEASES" -v fixed_now=1787950800 </dev/null
ss_spec_assert_file_contains "$JSON" '"id":"unknown","mac":"","ip":"","hostname":"Unknown devices","identified":false,"queries":4,"blocked":1'
ss_spec_assert_file_not_contains "$JSON" '"id":"other"'
ss_spec_assert_file_contains "$JSON" '"sanitized_hours":0'
ss_spec_assert_file_contains "$JSON" '"untracked_queries":0,"untracked_blocked":0'
ss_spec_assert_file_contains "$STATE" "$(printf 'source\tepoch\tudp\t128\t1\t0\t10\t3\t1\t0\t0\t5')"
)