mirror of
https://github.com/kiddin9/op-packages.git
synced 2026-11-15 04:53:00 +08:00
🎉 Sync 2026-11-02 12:48:14
This commit is contained in:
github-actions[bot] committed
1 parent
42d5cbfa43
commit
db798bdf5e
53 files changed
+1895
-568
No files matched your search
@@ -1,5 +1,48 @@
|
||||
# Changelog
|
||||
|
||||
## [0.3.26-r1] - 2026-11-02
|
||||
|
||||
- Bump version for release.
|
||||
|
||||
## [0.3.25-r4] - 2026-11-02
|
||||
|
||||
### Improved
|
||||
|
||||
- Reduce large temporary files during blocklist refreshes and clean up intermediate files after each refresh to lower memory usage on routers.
|
||||
- Add a lightweight statistics summary method for dashboards that only need aggregate information while preserving the existing full statistics response.
|
||||
|
||||
### Tests
|
||||
|
||||
- Verify custom allow/block rule precedence, generated blocklist output, and compatibility between the new statistics summary and existing detailed queries.
|
||||
|
||||
## [0.3.25-r3] - 2026-11-01
|
||||
|
||||
### Improved
|
||||
|
||||
- Skip expired-statistics cleanup when the retention boundary is unchanged, and avoid recomputing aggregate totals when statistics have not changed.
|
||||
- Skip temporary devices with no records in the upload period when preparing recent statistics uploads.
|
||||
- Batch status initialization and refresh-result updates to reduce repeated reads and writes of the status file.
|
||||
|
||||
### Tests
|
||||
|
||||
- Verify that retention behavior and upload output remain unchanged and that batched status updates preserve all values.
|
||||
|
||||
## [0.3.25-r2] - 2026-11-01
|
||||
|
||||
### Fixed
|
||||
|
||||
- Harden locking to prevent concurrent statistics collector and automatic refresh runs.
|
||||
- Avoid unnecessary DNS service restarts when blocklist downloads or preparation fail, preserving existing protection.
|
||||
- Prevent status file corruption and lost updates during concurrent status writes.
|
||||
|
||||
### Improved
|
||||
|
||||
- Use built-in OpenWrt tools for locking without requiring a separate `flock` command.
|
||||
|
||||
### Tests
|
||||
|
||||
- Test lock contention, stale-lock recovery, lock ownership, concurrent status updates, and avoidance of DNS restarts after pre-installation failures.
|
||||
|
||||
## [0.3.25-r1] - 2026-10-10
|
||||
|
||||
- Bump version for release.
|
||||
|
||||
+3
-2
@@ -6,8 +6,8 @@
|
||||
include $(TOPDIR)/rules.mk
|
||||
|
||||
PKG_NAME:=safeshield
|
||||
PKG_VERSION:=0.3.25
|
||||
PKG_RELEASE:=4
|
||||
PKG_VERSION:=0.3.26
|
||||
PKG_RELEASE:=5
|
||||
|
||||
PKG_MAINTAINER:=Beomjun Kang <kals323@gmail.com>
|
||||
PKG_LICENSE:=GPL-3.0-or-later
|
||||
@@ -80,6 +80,7 @@ define Package/safeshield/install
|
||||
$(INSTALL_BIN) ./files/usr/lib/safeshield/log.sh $(1)/usr/lib/safeshield/log.sh
|
||||
$(INSTALL_BIN) ./files/usr/lib/safeshield/status.sh $(1)/usr/lib/safeshield/status.sh
|
||||
$(INSTALL_BIN) ./files/usr/lib/safeshield/utils.sh $(1)/usr/lib/safeshield/utils.sh
|
||||
$(INSTALL_DATA) ./files/usr/lib/safeshield/locking.sh $(1)/usr/lib/safeshield/locking.sh
|
||||
$(INSTALL_DATA) ./files/usr/lib/safeshield/config.sh $(1)/usr/lib/safeshield/config.sh
|
||||
$(INSTALL_BIN) ./files/usr/lib/safeshield/identity.sh $(1)/usr/lib/safeshield/identity.sh
|
||||
$(INSTALL_DATA) ./files/usr/lib/safeshield/dns.sh $(1)/usr/lib/safeshield/dns.sh
|
||||
|
||||
@@ -399,6 +399,7 @@ Available methods:
|
||||
safeshield.status
|
||||
safeshield.config
|
||||
safeshield.statistics
|
||||
safeshield.statistics_summary
|
||||
safeshield.config_update
|
||||
safeshield.set_enabled
|
||||
safeshield.refresh
|
||||
@@ -420,6 +421,18 @@ ubus call safeshield statistics
|
||||
/etc/init.d/safeshield statistics
|
||||
```
|
||||
|
||||
For dashboards that need collector health and aggregate counters only, use:
|
||||
|
||||
```sh
|
||||
ubus call safeshield statistics_summary
|
||||
```
|
||||
|
||||
The summary retains configuration, service health, source details and totals,
|
||||
but omits the potentially large `hourly` and `devices` arrays. The original
|
||||
`statistics` response remains unchanged (schema v3). Both methods read the
|
||||
current statistics snapshot; the summary specifically reduces RPC-side array
|
||||
normalization and JSON response size, not the snapshot file read itself.
|
||||
|
||||
Statistics are collected from dnsmasq cumulative SafeShield UBus counters. Raw queried domains
|
||||
are never written to the statistics state. Loopback (`127.0.0.0/8` and `::1`)
|
||||
queries are excluded so SafeShield's own DNS health checks do not inflate user
|
||||
|
||||
@@ -16,7 +16,7 @@ IPKG_INSTROOT="${IPKG_INSTROOT:-}"
|
||||
export IPKG_INSTROOT
|
||||
|
||||
readonly PKG_NAME='safeshield'
|
||||
readonly PKG_VERSION='0.3.25-r1'
|
||||
readonly PKG_VERSION='0.3.26-r1'
|
||||
|
||||
if type extra_command >/dev/null 2>&1; then
|
||||
extra_command 'apply_local_rules' 'Apply local allow/block rules using the cached Hub artifact'
|
||||
|
||||
@@ -876,7 +876,80 @@ ss_merge_sorted_api_sources() {
|
||||
sort -m -u "$@" >"$out"
|
||||
}
|
||||
|
||||
# Build the final blocked-rule stream without two full-sized domain copies.
|
||||
# Keep the ancestor/equal precedence used by ss_filter_domains_shadowed_by_ancestors:
|
||||
# local allow > local block > Hub allow > Hub block.
|
||||
ss_write_effective_block_rules() {
|
||||
local shadow_file="$1"
|
||||
local local_blocks="$2"
|
||||
local api_blocks="$3"
|
||||
local output="$4"
|
||||
local prefix="$5"
|
||||
local suffix="$6"
|
||||
|
||||
# With no effective allow rules there is nothing to shadow the Hub blocks.
|
||||
# Avoid building an AWK lookup table and splitting every domain, but keep
|
||||
# the same local-first ordering and input validation as the normal path.
|
||||
if [ ! -s "$shadow_file" ]; then
|
||||
awk -v local_blocks="$local_blocks" -v prefix="$prefix" -v suffix="$suffix" '
|
||||
FILENAME == local_blocks {
|
||||
if ($0 != "") print prefix $0 suffix
|
||||
next
|
||||
}
|
||||
/^[a-z0-9._-]+$/ { print prefix $0 suffix }
|
||||
' "$local_blocks" "$api_blocks" >"$output"
|
||||
return $?
|
||||
fi
|
||||
|
||||
awk -v shadow_file="$shadow_file" -v local_blocks="$local_blocks" \
|
||||
-v prefix="$prefix" -v suffix="$suffix" '
|
||||
FILENAME == shadow_file {
|
||||
if ($0 != "") shadow[$0] = 1
|
||||
next
|
||||
}
|
||||
FILENAME == local_blocks {
|
||||
if ($0 != "") print prefix $0 suffix
|
||||
next
|
||||
}
|
||||
/^[a-z0-9._-]+$/ {
|
||||
if ($0 in shadow) next
|
||||
n = split($0, labels, ".")
|
||||
ancestor = labels[n]
|
||||
for (i = n - 1; i >= 1; i--) {
|
||||
ancestor = labels[i] "." ancestor
|
||||
if (ancestor in shadow) next
|
||||
}
|
||||
print prefix $0 suffix
|
||||
}
|
||||
' "$shadow_file" "$local_blocks" "$api_blocks" >"$output"
|
||||
}
|
||||
|
||||
# Merge scratch files live on tmpfs. API shards and local lists are retained
|
||||
# for refresh retry/reuse, but these derived copies are never reused.
|
||||
ss_cleanup_merge_scratch() {
|
||||
rm -f \
|
||||
"${SS_TMP_DIR}/api-blocks.domains" \
|
||||
"${SS_TMP_DIR}/api-allows.domains" \
|
||||
"${SS_TMP_DIR}/effective-local-blocks.domains" \
|
||||
"${SS_TMP_DIR}/effective-api-allows.domains" \
|
||||
"${SS_TMP_DIR}/effective-allows.domains" \
|
||||
"${SS_TMP_DIR}/effective-api-blocks.domains" \
|
||||
"${SS_TMP_DIR}/final-blocks.domains"
|
||||
}
|
||||
|
||||
# Always release derived tmpfs storage, including when validation fails.
|
||||
ss_merge_lists() {
|
||||
local result=0
|
||||
# Firmware refresh uses the C locale. Enforce it here too so direct
|
||||
# calls (including CI) retain the exact same bytewise domain order.
|
||||
local LC_ALL=C
|
||||
export LC_ALL
|
||||
ss_merge_lists_inner "$@" || result=$?
|
||||
ss_cleanup_merge_scratch
|
||||
return "$result"
|
||||
}
|
||||
|
||||
ss_merge_lists_inner() {
|
||||
local final
|
||||
local api_blocks="${SS_TMP_DIR}/api-blocks.domains"
|
||||
local api_allows="${SS_TMP_DIR}/api-allows.domains"
|
||||
@@ -885,10 +958,9 @@ ss_merge_lists() {
|
||||
local effective_local_blocks="${SS_TMP_DIR}/effective-local-blocks.domains"
|
||||
local effective_api_allows="${SS_TMP_DIR}/effective-api-allows.domains"
|
||||
local effective_allows="${SS_TMP_DIR}/effective-allows.domains"
|
||||
local effective_api_blocks="${SS_TMP_DIR}/effective-api-blocks.domains"
|
||||
local final_blocks="${SS_TMP_DIR}/final-blocks.domains"
|
||||
local final_size_kb
|
||||
local valid_count
|
||||
local rule_prefix rule_suffix
|
||||
|
||||
final="$(ss_blocklist_tmp_path)" || return 1
|
||||
: >"$final" || return 1
|
||||
@@ -929,39 +1001,39 @@ ss_merge_lists() {
|
||||
rm -f "$final"
|
||||
return 1
|
||||
}
|
||||
ss_filter_domains_shadowed_by_ancestors "$effective_allows" "$api_blocks" "$effective_api_blocks" || {
|
||||
# Produce final rule lines directly rather than materializing both a full
|
||||
# filtered API blocklist and a second combined domain list. The input API
|
||||
# shards were already merged and sorted above; the final sort preserves the
|
||||
# exact dnsmasq rule order/deduplication of the old pipeline.
|
||||
if ss_dnsmasq_safeshield_block_supported; then
|
||||
rule_prefix='safeshield-block=/'
|
||||
rule_suffix='/'
|
||||
else
|
||||
rule_prefix='address=/'
|
||||
rule_suffix='/#'
|
||||
fi
|
||||
ss_write_effective_block_rules \
|
||||
"$effective_allows" "$effective_local_blocks" "$api_blocks" \
|
||||
"$final" "$rule_prefix" "$rule_suffix" || {
|
||||
rm -f "$final"
|
||||
return 1
|
||||
}
|
||||
{
|
||||
[ -f "$effective_local_blocks" ] && cat "$effective_local_blocks"
|
||||
[ -f "$effective_api_blocks" ] && cat "$effective_api_blocks"
|
||||
} | sort -u >"$final_blocks" || {
|
||||
# Sort by the domain field (between slashes), not the whole dnsmasq rule.
|
||||
# Sorting full rules changes legacy ordering for domain-prefix pairs such
|
||||
# as foo and foo-bar, causing otherwise identical refreshes to differ.
|
||||
# The domain field cannot contain '/' after normalization. -o keeps the
|
||||
# output in-place without allocating a second full-size domain copy.
|
||||
# Bound GNU sort's in-memory buffer on low-memory routers. External sort
|
||||
# runs may use more temporary disk space; the final order is unchanged.
|
||||
sort -S 1M -u -t/ -k2,2 -o "$final" "$final" || {
|
||||
rm -f "$final"
|
||||
return 1
|
||||
}
|
||||
|
||||
# Preserve the cancellation point from the original merge pipeline.
|
||||
ss_should_stop && {
|
||||
rm -f "$final"
|
||||
return 130
|
||||
}
|
||||
|
||||
if [ -s "$final_blocks" ]; then
|
||||
if ss_dnsmasq_safeshield_block_supported; then
|
||||
awk '{ print "safeshield-block=/" $0 "/" }' "$final_blocks" >"$final" || {
|
||||
rm -f "$final"
|
||||
return 1
|
||||
}
|
||||
else
|
||||
# Stock dnsmasq does not understand the SafeShield extension. Preserve
|
||||
# protection with the standard address rule and leave statistics off.
|
||||
awk '{ print "address=/" $0 "/#" }' "$final_blocks" >"$final" || {
|
||||
rm -f "$final"
|
||||
return 1
|
||||
}
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -s "$effective_allows" ]; then
|
||||
awk '{ print "server=/" $0 "/#" }' "$effective_allows" >>"$final" || {
|
||||
rm -f "$final"
|
||||
|
||||
@@ -46,7 +46,6 @@ readonly SS_MAX_ARTIFACT_SOURCES=16
|
||||
readonly SS_PREV_BLOCKLIST_GZ="/tmp/safeshield.prev.blocklist.gz"
|
||||
readonly SS_RUNTIME_OUT="${SS_TMP_DIR}/runtime.out"
|
||||
readonly SS_REFRESH_LOCK='/var/lock/safeshield-refresh.lock'
|
||||
readonly SS_REFRESH_LOCK_FD=307
|
||||
readonly SS_LOCAL_APPLY_STATE="${SS_TMP_DIR}/local-apply.state"
|
||||
readonly SS_STATISTICS_DIR="${SS_TMP_DIR}/statistics"
|
||||
readonly SS_STATISTICS_STATE_FILE="${SS_STATISTICS_DIR}/state.tsv"
|
||||
@@ -444,7 +443,6 @@ safeshield_apply_local_rules() {
|
||||
*)
|
||||
log_error "Failed to merge cached Hub artifact with local rules"
|
||||
ss_mark_local_apply_failure "local_merge_failed"
|
||||
ss_restore_and_restart
|
||||
ss_refresh_lock_close
|
||||
return 1
|
||||
;;
|
||||
@@ -647,9 +645,7 @@ safeshield_force_download() {
|
||||
*)
|
||||
failure_code="$(ss_resolve_error_code)"
|
||||
ss_status_mark_failure "$failure_code"
|
||||
if [ "$failure_code" != "safeshield_upgrade_required" ]; then
|
||||
ss_restore_and_restart
|
||||
fi
|
||||
# No blocklist replacement has happened; keep the running DNS service.
|
||||
ss_refresh_lock_close
|
||||
return 1
|
||||
;;
|
||||
@@ -672,9 +668,7 @@ safeshield_force_download() {
|
||||
*)
|
||||
failure_code="$(ss_artifact_download_error_code)"
|
||||
ss_status_mark_failure "$failure_code"
|
||||
if [ "$failure_code" != "safeshield_upgrade_required" ]; then
|
||||
ss_restore_and_restart
|
||||
fi
|
||||
# No blocklist replacement has happened; keep the running DNS service.
|
||||
ss_refresh_lock_close
|
||||
return 1
|
||||
;;
|
||||
@@ -689,7 +683,6 @@ safeshield_force_download() {
|
||||
ss_prepare_local_rule_files || {
|
||||
log_error "Failed to build allowlist"
|
||||
ss_status_mark_failure "allowlist_build_failed"
|
||||
ss_restore_and_restart
|
||||
ss_refresh_lock_close
|
||||
return 1
|
||||
}
|
||||
@@ -711,7 +704,6 @@ safeshield_force_download() {
|
||||
*)
|
||||
log_error "Failed to merge API artifact with local overrides"
|
||||
ss_status_mark_failure "merge_failed"
|
||||
ss_restore_and_restart
|
||||
ss_refresh_lock_close
|
||||
return 1
|
||||
;;
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
# shellcheck shell=ash
|
||||
|
||||
# Directory locks work with the OpenWrt BusyBox base utilities; flock(1) is
|
||||
# intentionally not required. A pid-less directory is treated as initializing
|
||||
# (or abandoned) and must age before it can be reclaimed.
|
||||
ss_lock_pid_alive() {
|
||||
case "${1:-}" in
|
||||
'' | *[!0-9]* | 0) return 1 ;;
|
||||
esac
|
||||
kill -0 "$1" 2>/dev/null
|
||||
}
|
||||
|
||||
ss_lock_stale() {
|
||||
local path="$1" owner='' modified='' now=''
|
||||
|
||||
[ -d "$path" ] || return 1
|
||||
if [ -r "$path/pid" ]; then
|
||||
IFS= read -r owner <"$path/pid" || true
|
||||
# A live owner always wins, even if the directory appears old.
|
||||
ss_lock_pid_alive "$owner" && return 1
|
||||
# Never reclaim an invalid pid that could be mid-write without a grace period.
|
||||
case "$owner" in
|
||||
'' | *[!0-9]* | 0) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# stat is available in the supported BusyBox builds. If it is missing,
|
||||
# fail closed rather than destroying a possibly initializing lock.
|
||||
modified="$(stat -c %Y "$path" 2>/dev/null)" || return 1
|
||||
now="$(date +%s 2>/dev/null)" || return 1
|
||||
case "$modified:$now" in
|
||||
*[!0-9:]* | :* | *:) return 1 ;;
|
||||
esac
|
||||
[ "$((now - modified))" -ge 30 ] 2>/dev/null
|
||||
}
|
||||
|
||||
ss_lock_reclaim() {
|
||||
local path="$1" guard="${1}.reclaim" owner=''
|
||||
|
||||
# Only one contender may inspect/remove a stale directory at a time.
|
||||
mkdir "$guard" 2>/dev/null || return 1
|
||||
if ! ss_lock_stale "$path"; then
|
||||
rmdir "$guard" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
if [ -r "$path/pid" ]; then
|
||||
IFS= read -r owner <"$path/pid" || true
|
||||
fi
|
||||
if ss_lock_pid_alive "$owner"; then
|
||||
rmdir "$guard" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
# A foreign file makes this fail safely instead of deleting arbitrary data.
|
||||
rm -f "$path/pid" 2>/dev/null || true
|
||||
if ! rmdir "$path" 2>/dev/null; then
|
||||
rmdir "$guard" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
rmdir "$guard" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
|
||||
ss_lock_try() {
|
||||
local path="$1" attempt=0
|
||||
[ -n "$path" ] || return 1
|
||||
mkdir -p "${path%/*}" || return 1
|
||||
while [ "$attempt" -lt 2 ]; do
|
||||
# Prevent new acquisitions while stale-lock removal is in progress.
|
||||
if [ -d "${path}.reclaim" ]; then
|
||||
# A crashed reclaimer must not block all future acquisitions forever.
|
||||
# Refuse to touch a recent guard still owned by an active reclaimer.
|
||||
ss_lock_stale "${path}.reclaim" && rmdir "${path}.reclaim" 2>/dev/null || return 1
|
||||
fi
|
||||
if mkdir "$path" 2>/dev/null; then
|
||||
if ! printf '%s\n' "$$" >"$path/pid"; then
|
||||
rmdir "$path" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
if [ -e "${path}.reclaim" ]; then
|
||||
ss_lock_release "$path"
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
fi
|
||||
[ "$attempt" -eq 0 ] || break
|
||||
ss_lock_reclaim "$path" || return 1
|
||||
attempt=$((attempt + 1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
ss_lock_wait() {
|
||||
local path="$1" timeout="${2:-10}" stop_cb="${3:-}" waited=0
|
||||
case "$timeout" in
|
||||
'' | *[!0-9]*) timeout=10 ;;
|
||||
esac
|
||||
while ! ss_lock_try "$path"; do
|
||||
if [ "$waited" -ge "$timeout" ]; then
|
||||
return 1
|
||||
fi
|
||||
if [ -n "$stop_cb" ] && "$stop_cb"; then
|
||||
return 130
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
ss_lock_release() {
|
||||
local path="$1" owner=''
|
||||
[ -n "$path" ] && [ -r "$path/pid" ] || return 0
|
||||
IFS= read -r owner <"$path/pid" || true
|
||||
[ "$owner" = "$$" ] || return 1
|
||||
# Do not remove someone else's data even if the lock has extra contents.
|
||||
rm -f "$path/pid" || return 1
|
||||
rmdir "$path" 2>/dev/null
|
||||
}
|
||||
@@ -30,6 +30,16 @@ function device_bucket_key(key, bucket) {
|
||||
|
||||
function prune_buckets(now, cutoff, bucket, composite, parts) {
|
||||
cutoff = hour_start(now) - ((retention_hours - 1) * 3600)
|
||||
# Existing snapshots only add counters to the active hour. Repeating the
|
||||
# full expired-bucket sweep every minute does no work until the cutoff
|
||||
# advances. A collector restart starts with an empty cache and prunes once.
|
||||
if (cutoff == last_prune_cutoff) {
|
||||
return
|
||||
}
|
||||
last_prune_cutoff = cutoff
|
||||
# A new retention window can invalidate cached totals even without DNS
|
||||
# traffic; the full aggregate is rebuilt at the next snapshot.
|
||||
aggregate_dirty = 1
|
||||
for (bucket in queries) {
|
||||
if ((bucket + 0) < cutoff) {
|
||||
delete queries[bucket]
|
||||
@@ -136,6 +146,7 @@ function register_device(key, mac, ip, hostname, client_id, duid, selected, o
|
||||
# device record structurally complete, this avoids passing a missing awk
|
||||
# associative-array element to helper functions during serialization.
|
||||
device_seen[selected] = 1
|
||||
aggregate_dirty = 1
|
||||
device_mac[selected] = ""
|
||||
device_ip[selected] = ""
|
||||
device_hostname[selected] = ""
|
||||
|
||||
@@ -282,6 +282,7 @@ function migrate_device(from_key, to_key, mac, ip, hostname, client_id, duid,
|
||||
if (!journal_replaying) {
|
||||
journal_deleted_device[from_key] = 1
|
||||
}
|
||||
aggregate_dirty = 1
|
||||
remove_device_metadata(from_key)
|
||||
selected = register_device(to_key, mac, ip, hostname, client_id, duid)
|
||||
if (selected == "") {
|
||||
@@ -533,6 +534,7 @@ function record_device(ip, query_delta, blocked_delta, now, key, bucket, comp
|
||||
|
||||
bucket = hour_start(now)
|
||||
composite = device_bucket_key(key, bucket)
|
||||
aggregate_dirty = 1
|
||||
device_hour_queries[composite] += 0
|
||||
device_hour_blocked[composite] += 0
|
||||
if (query_delta > 0) {
|
||||
|
||||
@@ -53,6 +53,12 @@ function save_json_range(path, now, first_hour, last_hour, include_empty_devices
|
||||
if (!is_unknown_output_device(key)) {
|
||||
continue
|
||||
}
|
||||
# An upload contains only a short recent window. Ignore provisional
|
||||
# devices whose retained buckets fall completely outside that window.
|
||||
if ((device_last_bucket[key] + 0) < first_hour ||
|
||||
(device_first_bucket[key] + 0) > last_hour) {
|
||||
continue
|
||||
}
|
||||
for (bucket = first_hour; bucket <= last_hour; bucket += 3600) {
|
||||
composite = device_bucket_key(key, bucket)
|
||||
if (!((composite in device_hour_queries) || (composite in device_hour_blocked))) {
|
||||
@@ -228,9 +234,16 @@ function save_snapshot(now, force_persistent, force_snapshot, serialize_snaps
|
||||
serialize_snapshot = snapshot_dirty || force_snapshot || (force_persistent && persistence_configured)
|
||||
if (serialize_snapshot) {
|
||||
prune_buckets(now)
|
||||
recompute_totals()
|
||||
# Error-only and unchanged UBus snapshots still refresh source metadata
|
||||
# but do not change retained counters. Avoid reaggregating 128 devices
|
||||
# across up to 168 hours unless counters, identities, or cutoff changed.
|
||||
if (aggregate_dirty) {
|
||||
recompute_totals()
|
||||
aggregate_dirty = 0
|
||||
}
|
||||
if (compact_stale_provisional_devices(now) > 0) {
|
||||
recompute_totals()
|
||||
aggregate_dirty = 0
|
||||
}
|
||||
updated_at = now
|
||||
if (!ensure_snapshot_sequence_reservation(now)) {
|
||||
|
||||
@@ -9,6 +9,7 @@ function record_event(query_delta, blocked_delta, client_ip, now, bucket) {
|
||||
now = current_time()
|
||||
bucket = hour_start(now)
|
||||
snapshot_dirty = 1
|
||||
aggregate_dirty = 1
|
||||
queries[bucket] += 0
|
||||
blocked[bucket] += 0
|
||||
|
||||
@@ -110,6 +111,7 @@ function apply_source_snapshot( now, bucket, had_source, same_epoch, query_de
|
||||
}
|
||||
|
||||
if (query_delta > 0 || blocked_delta > 0) {
|
||||
aggregate_dirty = 1
|
||||
queries[bucket] += 0
|
||||
blocked[bucket] += 0
|
||||
queries[bucket] += query_delta
|
||||
@@ -184,6 +186,9 @@ BEGIN {
|
||||
}
|
||||
event_index = 0
|
||||
snapshot_dirty = 0
|
||||
# BEGIN restores state and reconciles device identities before the first
|
||||
# snapshot, so never trust persisted aggregates as an in-memory cache.
|
||||
aggregate_dirty = 1
|
||||
device_count = 0
|
||||
devices_truncated = 0
|
||||
last_lease_refresh = 0
|
||||
|
||||
@@ -62,6 +62,19 @@ else if (cmd == 'set') {
|
||||
|
||||
obj.data[key] = value;
|
||||
}
|
||||
else if (cmd == 'set_many') {
|
||||
// Reject incomplete input before modifying any field; one JSON write is
|
||||
// committed only after every key/value pair has been validated.
|
||||
if (length(ARGV) < 4 || (length(ARGV) - 2) % 2 != 0)
|
||||
exit(2);
|
||||
|
||||
for (let i = 2; i < length(ARGV); i += 2) {
|
||||
if (!ARGV[i])
|
||||
exit(2);
|
||||
}
|
||||
for (let i = 2; i < length(ARGV); i += 2)
|
||||
obj.data[ARGV[i]] = ARGV[i + 1];
|
||||
}
|
||||
else if (cmd == 'add_error') {
|
||||
let code = ARGV[2];
|
||||
|
||||
|
||||
@@ -10,26 +10,19 @@ ss_status_exec() {
|
||||
}
|
||||
|
||||
ss_status_apply() {
|
||||
local tmp="${RUNNING_STATUS_FILE}.tmp"
|
||||
local tmp="${RUNNING_STATUS_FILE}.tmp.$$" rc=0
|
||||
|
||||
mkdir -p "${RUNNING_STATUS_FILE%/*}" || return 1
|
||||
json_lock_open || return 1
|
||||
|
||||
ss_status_exec "$@" >"${tmp}" || {
|
||||
rm -f "${tmp}"
|
||||
return 1
|
||||
}
|
||||
|
||||
[ -s "${tmp}" ] || {
|
||||
rm -f "${tmp}"
|
||||
return 1
|
||||
}
|
||||
|
||||
mv -f "${tmp}" "${RUNNING_STATUS_FILE}" || {
|
||||
rm -f "${tmp}"
|
||||
return 1
|
||||
}
|
||||
|
||||
return 0
|
||||
ss_status_exec "$@" >"$tmp" || rc=1
|
||||
[ "$rc" -eq 0 ] && [ -s "$tmp" ] || rc=1
|
||||
if [ "$rc" -eq 0 ]; then
|
||||
mv -f "$tmp" "$RUNNING_STATUS_FILE" || rc=1
|
||||
fi
|
||||
[ "$rc" -eq 0 ] || rm -f "$tmp"
|
||||
json_lock_close
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
ss_now() {
|
||||
@@ -40,66 +33,77 @@ ss_status_set_now() {
|
||||
ss_status_set "$1" "$(ss_now)"
|
||||
}
|
||||
|
||||
# A single status-store invocation updates all pairs while holding the same
|
||||
# JSON lock. This avoids spawning ucode once per field during a refresh reset.
|
||||
ss_status_set_many() {
|
||||
[ "$#" -gt 0 ] || return 0
|
||||
ss_status_apply set_many "$@" >/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
ss_status_reset_health_fields() {
|
||||
ss_status_set health_dnsmasq_binary ""
|
||||
ss_status_set health_dnsmasq_version ""
|
||||
ss_status_set health_dnsmasq_features ""
|
||||
ss_status_set health_dnsmasq_confdir ""
|
||||
ss_status_set health_dnsmasq_initial_restart ""
|
||||
ss_status_set health_dnsmasq_final_restart ""
|
||||
ss_status_set health_dns_runtime ""
|
||||
ss_status_set health_blocklist_verify ""
|
||||
ss_status_set health_min_valid_line_count ""
|
||||
ss_status_set health_max_file_size ""
|
||||
ss_status_set health_api_resolve ""
|
||||
ss_status_set health_artifact_download ""
|
||||
ss_status_set health_artifact_sha256 ""
|
||||
ss_status_set dnsmasq_version ""
|
||||
ss_status_set dnsmasq_min_version "${SS_MIN_DNSMASQ_VERSION:-2.93}"
|
||||
ss_status_set_many \
|
||||
health_dnsmasq_binary "" \
|
||||
health_dnsmasq_version "" \
|
||||
health_dnsmasq_features "" \
|
||||
health_dnsmasq_confdir "" \
|
||||
health_dnsmasq_initial_restart "" \
|
||||
health_dnsmasq_final_restart "" \
|
||||
health_dns_runtime "" \
|
||||
health_blocklist_verify "" \
|
||||
health_min_valid_line_count "" \
|
||||
health_max_file_size "" \
|
||||
health_api_resolve "" \
|
||||
health_artifact_download "" \
|
||||
health_artifact_sha256 "" \
|
||||
dnsmasq_version "" \
|
||||
dnsmasq_min_version "${SS_MIN_DNSMASQ_VERSION:-2.93}"
|
||||
}
|
||||
|
||||
ss_status_reset_blocklist_fields() {
|
||||
ss_status_set blocklist_installed "0"
|
||||
ss_status_set blocklist_file_size_kb "0"
|
||||
ss_status_set blocklist_verification_ok "0"
|
||||
ss_status_set blocklist_test_domain ""
|
||||
ss_status_set blocklist_test_domain_sample_count "0"
|
||||
ss_status_set blocklist_test_domain_success_count "0"
|
||||
ss_status_set blocklist_test_domains ""
|
||||
ss_status_set blocklist_backup_available "0"
|
||||
ss_status_set_many \
|
||||
blocklist_installed "0" \
|
||||
blocklist_file_size_kb "0" \
|
||||
blocklist_verification_ok "0" \
|
||||
blocklist_test_domain "" \
|
||||
blocklist_test_domain_sample_count "0" \
|
||||
blocklist_test_domain_success_count "0" \
|
||||
blocklist_test_domains "" \
|
||||
blocklist_backup_available "0"
|
||||
}
|
||||
|
||||
ss_status_reset_artifact_fields() {
|
||||
ss_status_set entitlement_plan ""
|
||||
ss_status_set entitlement_status ""
|
||||
ss_status_set physical_fingerprint ""
|
||||
ss_status_set fingerprint_version ""
|
||||
ss_status_set identity_provider ""
|
||||
ss_status_set identity_source ""
|
||||
ss_status_set identity_strength ""
|
||||
ss_status_set identity_profile ""
|
||||
ss_status_set device_code ""
|
||||
ss_status_set device_code_source ""
|
||||
ss_status_set installation_id ""
|
||||
ss_status_set device_profile ""
|
||||
ss_status_set artifact_tier ""
|
||||
ss_status_set artifact_version ""
|
||||
ss_status_set artifact_sha256 ""
|
||||
ss_status_set artifact_unique_domains "0"
|
||||
ss_status_set artifact_rules "0"
|
||||
ss_status_set artifact_download_url_present "0"
|
||||
ss_status_set artifact_source_count "0"
|
||||
ss_status_set artifact_block_source_count "0"
|
||||
ss_status_set artifact_allow_source_count "0"
|
||||
ss_status_set_many \
|
||||
entitlement_plan "" \
|
||||
entitlement_status "" \
|
||||
physical_fingerprint "" \
|
||||
fingerprint_version "" \
|
||||
identity_provider "" \
|
||||
identity_source "" \
|
||||
identity_strength "" \
|
||||
identity_profile "" \
|
||||
device_code "" \
|
||||
device_code_source "" \
|
||||
installation_id "" \
|
||||
device_profile "" \
|
||||
artifact_tier "" \
|
||||
artifact_version "" \
|
||||
artifact_sha256 "" \
|
||||
artifact_unique_domains "0" \
|
||||
artifact_rules "0" \
|
||||
artifact_download_url_present "0" \
|
||||
artifact_source_count "0" \
|
||||
artifact_block_source_count "0" \
|
||||
artifact_allow_source_count "0"
|
||||
}
|
||||
|
||||
ss_status_mark_failure() {
|
||||
local code="$1"
|
||||
|
||||
ss_status_set status "error"
|
||||
ss_status_set_now last_failure
|
||||
ss_status_set last_result "error"
|
||||
ss_status_set last_error_code "$code"
|
||||
ss_status_set_many \
|
||||
status "error" \
|
||||
last_failure "$(ss_now)" \
|
||||
last_result "error" \
|
||||
last_error_code "$code"
|
||||
ss_status_add_error "$code"
|
||||
}
|
||||
|
||||
@@ -109,14 +113,15 @@ ss_status_mark_success() {
|
||||
|
||||
now="$(ss_now)"
|
||||
ss_status_apply clear_messages >/dev/null 2>&1 || true
|
||||
ss_status_set status "ready"
|
||||
ss_status_set stage "done"
|
||||
ss_status_set last_success "$now"
|
||||
ss_status_set last_result "success"
|
||||
ss_status_set last_error_code ""
|
||||
|
||||
if is_valid_integer "$interval" && [ "$interval" -gt 0 ] 2>/dev/null; then
|
||||
ss_status_set next_refresh_at "$((now + interval))"
|
||||
ss_status_set_many \
|
||||
status "ready" stage "done" last_success "$now" \
|
||||
last_result "success" last_error_code "" \
|
||||
next_refresh_at "$((now + interval))"
|
||||
else
|
||||
ss_status_set_many \
|
||||
status "ready" stage "done" last_success "$now" \
|
||||
last_result "success" last_error_code ""
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -131,9 +136,10 @@ ss_status_prepare_refresh() {
|
||||
is_valid_integer "$last_local_apply_failure" || last_local_apply_failure=0
|
||||
|
||||
ss_status_reset
|
||||
ss_status_set last_success "$last_success"
|
||||
ss_status_set last_local_apply "$last_local_apply"
|
||||
ss_status_set last_local_apply_failure "$last_local_apply_failure"
|
||||
ss_status_set_many \
|
||||
last_success "$last_success" \
|
||||
last_local_apply "$last_local_apply" \
|
||||
last_local_apply_failure "$last_local_apply_failure"
|
||||
}
|
||||
|
||||
ss_status_set() {
|
||||
@@ -159,22 +165,14 @@ ss_status_add_warning() {
|
||||
}
|
||||
|
||||
ss_status_reset() {
|
||||
rm -f "${RUNNING_STATUS_FILE}" "${RUNNING_STATUS_FILE}.tmp"
|
||||
|
||||
ss_status_apply reset >/dev/null 2>&1 || return 1
|
||||
|
||||
ss_status_set status "idle"
|
||||
ss_status_set stage ""
|
||||
ss_status_set valid_line_count "0"
|
||||
|
||||
ss_status_set last_result "idle"
|
||||
ss_status_set last_error_code ""
|
||||
ss_status_set last_attempt "0"
|
||||
ss_status_set last_success "0"
|
||||
ss_status_set last_failure "0"
|
||||
ss_status_set last_local_apply "0"
|
||||
ss_status_set last_local_apply_failure "0"
|
||||
ss_status_set next_refresh_at "0"
|
||||
ss_status_set_many \
|
||||
status "idle" stage "" valid_line_count "0" \
|
||||
last_result "idle" last_error_code "" \
|
||||
last_attempt "0" last_success "0" last_failure "0" \
|
||||
last_local_apply "0" last_local_apply_failure "0" \
|
||||
next_refresh_at "0"
|
||||
|
||||
ss_status_reset_health_fields
|
||||
ss_status_reset_blocklist_fields
|
||||
@@ -262,41 +260,15 @@ ss_restore_previous_blocklist() {
|
||||
}
|
||||
|
||||
ss_refresh_lock_open() {
|
||||
mkdir -p "${SS_REFRESH_LOCK%/*}" || return 1
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>\"${SS_REFRESH_LOCK}\"" || return 1
|
||||
flock -n "${SS_REFRESH_LOCK_FD}" || {
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>&-"
|
||||
return 1
|
||||
}
|
||||
ss_lock_try "${SS_REFRESH_LOCK}.d"
|
||||
}
|
||||
|
||||
ss_refresh_lock_open_wait() {
|
||||
local timeout="${1:-120}"
|
||||
local waited=0
|
||||
|
||||
is_valid_integer "$timeout" || timeout=120
|
||||
mkdir -p "${SS_REFRESH_LOCK%/*}" || return 1
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>\"${SS_REFRESH_LOCK}\"" || return 1
|
||||
|
||||
while ! flock -n "${SS_REFRESH_LOCK_FD}"; do
|
||||
if [ "$waited" -ge "$timeout" ] 2>/dev/null; then
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>&-" 2>/dev/null || true
|
||||
return 1
|
||||
fi
|
||||
|
||||
ss_should_stop && {
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>&-" 2>/dev/null || true
|
||||
return 130
|
||||
}
|
||||
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
|
||||
return 0
|
||||
ss_lock_wait "${SS_REFRESH_LOCK}.d" "$timeout" ss_should_stop
|
||||
}
|
||||
|
||||
ss_refresh_lock_close() {
|
||||
flock -u "${SS_REFRESH_LOCK_FD}" 2>/dev/null || true
|
||||
eval "exec ${SS_REFRESH_LOCK_FD}>&-" 2>/dev/null || true
|
||||
ss_lock_release "${SS_REFRESH_LOCK}.d"
|
||||
}
|
||||
@@ -1,7 +1,9 @@
|
||||
# shellcheck shell=ash
|
||||
|
||||
readonly LOCK_FD=309
|
||||
readonly RUNNING_STATUS_LOCK="/var/lock/${PKG_NAME}.lock"
|
||||
readonly RUNNING_STATUS_LOCK="/var/lock/${PKG_NAME}.lock.d"
|
||||
|
||||
# shellcheck disable=SC1090
|
||||
. "${SS_LOCKING_LIB:-${IPKG_INSTROOT:-}/usr/lib/safeshield/locking.sh}"
|
||||
readonly RUNNING_STATUS_FILE="/dev/shm/${PKG_NAME}.status.json"
|
||||
|
||||
command_exists() {
|
||||
@@ -21,17 +23,23 @@ is_greater_equal() {
|
||||
}
|
||||
|
||||
json_lock_open() {
|
||||
mkdir -p "${RUNNING_STATUS_LOCK%/*}" || return 1
|
||||
eval "exec ${LOCK_FD}>\"${RUNNING_STATUS_LOCK}\"" || return 1
|
||||
flock -x "${LOCK_FD}" || {
|
||||
eval "exec ${LOCK_FD}>&-"
|
||||
return 1
|
||||
}
|
||||
ss_lock_wait "$RUNNING_STATUS_LOCK" 10
|
||||
}
|
||||
|
||||
json_lock_close() {
|
||||
flock -u "${LOCK_FD}" 2>/dev/null || true
|
||||
eval "exec ${LOCK_FD}>&-" 2>/dev/null || true
|
||||
ss_lock_release "$RUNNING_STATUS_LOCK"
|
||||
}
|
||||
|
||||
_json_commit() {
|
||||
local tmp="${RUNNING_STATUS_FILE}.tmp.$$"
|
||||
json_dump >"$tmp" || {
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
}
|
||||
mv -f "$tmp" "$RUNNING_STATUS_FILE" || {
|
||||
rm -f "$tmp"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
_json_init_data_object() {
|
||||
@@ -62,7 +70,7 @@ _json_set() {
|
||||
|
||||
json_add_string "$key" "$value"
|
||||
json_select .. 2>/dev/null || true
|
||||
json_dump >"${RUNNING_STATUS_FILE}"
|
||||
_json_commit
|
||||
}
|
||||
|
||||
_json_add_message() {
|
||||
@@ -82,7 +90,7 @@ _json_add_message() {
|
||||
json_add_string info "$value"
|
||||
json_close_object
|
||||
json_select .. 2>/dev/null || true
|
||||
json_dump >"${RUNNING_STATUS_FILE}"
|
||||
_json_commit
|
||||
}
|
||||
|
||||
json() {
|
||||
|
||||
@@ -14,20 +14,26 @@ export IPKG_INSTROOT
|
||||
. /usr/lib/safeshield/core.sh
|
||||
|
||||
readonly SS_REFRESHD_LOCK='/var/lock/safeshield-refreshd.lock'
|
||||
readonly SS_REFRESHD_LOCK_FD=308
|
||||
refreshd_owned=0
|
||||
|
||||
ss_should_terminate=0
|
||||
ss_sleep_pid=''
|
||||
|
||||
refreshd_lock() {
|
||||
mkdir -p "${SS_REFRESHD_LOCK%/*}" || return 1
|
||||
eval "exec ${SS_REFRESHD_LOCK_FD}>\"${SS_REFRESHD_LOCK}\"" || return 1
|
||||
flock -n "${SS_REFRESHD_LOCK_FD}" || return 1
|
||||
ss_lock_try "${SS_REFRESHD_LOCK}.d" || return 1
|
||||
refreshd_owned=1
|
||||
}
|
||||
|
||||
refreshd_unlock() {
|
||||
flock -u "${SS_REFRESHD_LOCK_FD}" 2>/dev/null || true
|
||||
eval "exec ${SS_REFRESHD_LOCK_FD}>&-" 2>/dev/null || true
|
||||
[ "${refreshd_owned:-0}" -eq 1 ] || return 0
|
||||
ss_lock_release "${SS_REFRESHD_LOCK}.d"
|
||||
refreshd_owned=0
|
||||
}
|
||||
|
||||
refreshd_cleanup() {
|
||||
[ "${refreshd_owned:-0}" -eq 1 ] || return 0
|
||||
clear_next_refresh_at
|
||||
refreshd_unlock
|
||||
}
|
||||
|
||||
handle_term() {
|
||||
@@ -253,6 +259,6 @@ main() {
|
||||
}
|
||||
|
||||
trap "handle_term" TERM INT
|
||||
trap "refreshd_unlock; clear_next_refresh_at" EXIT
|
||||
trap "refreshd_cleanup" EXIT
|
||||
|
||||
main "$@"
|
||||
@@ -7,6 +7,7 @@ LC_ALL=C
|
||||
export LC_ALL
|
||||
umask 077
|
||||
|
||||
SS_STATSD_LOCK_LIB="${SS_STATSD_LOCK_LIB:-/usr/lib/safeshield/locking.sh}"
|
||||
SS_STATSD_FUNCTIONS_LIB="${SS_STATSD_FUNCTIONS_LIB:-/lib/functions.sh}"
|
||||
SS_STATSD_CORE_LIB="${SS_STATSD_CORE_LIB:-/usr/lib/safeshield/core.sh}"
|
||||
SS_STATSD_AWK_DIR="${SS_STATSD_AWK_DIR:-/usr/lib/safeshield/statistics}"
|
||||
@@ -24,6 +25,8 @@ SS_STATSD_PERSISTENT_COMPACT_INTERVAL_S="${SS_STATSD_PERSISTENT_COMPACT_INTERVAL
|
||||
. "$SS_STATSD_FUNCTIONS_LIB"
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_STATSD_CORE_LIB"
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_STATSD_LOCK_LIB"
|
||||
|
||||
stats_poller_pid=''
|
||||
stats_awk_pid=''
|
||||
@@ -74,7 +77,8 @@ stats_cleanup() {
|
||||
[ -n "$stats_fifo" ] && rm -f "$stats_fifo"
|
||||
|
||||
if [ "$stats_lock_owned" -eq 1 ] && [ -n "$stats_lock_dir" ]; then
|
||||
rm -rf "$stats_lock_dir"
|
||||
ss_lock_release "$stats_lock_dir" || true
|
||||
stats_lock_owned=0
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -85,30 +89,12 @@ stats_terminate() {
|
||||
}
|
||||
|
||||
stats_acquire_lock() {
|
||||
local owner_pid=''
|
||||
|
||||
stats_lock_dir="${SS_STATISTICS_DIR}/collector.lock"
|
||||
|
||||
if mkdir "$stats_lock_dir" 2>/dev/null; then
|
||||
stats_lock_owned=1
|
||||
printf '%s\n' "$$" >"${stats_lock_dir}/pid"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ -r "${stats_lock_dir}/pid" ]; then
|
||||
owner_pid="$(cat "${stats_lock_dir}/pid" 2>/dev/null || true)"
|
||||
fi
|
||||
|
||||
if stats_process_running "$owner_pid"; then
|
||||
log_warn "Statistics collector is already running (pid=${owner_pid})"
|
||||
if ! ss_lock_try "$stats_lock_dir"; then
|
||||
log_warn 'Statistics collector lock is already held or unavailable'
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_warn 'Removing stale statistics collector lock'
|
||||
rm -rf "$stats_lock_dir" || return 1
|
||||
mkdir "$stats_lock_dir" || return 1
|
||||
stats_lock_owned=1
|
||||
printf '%s\n' "$$" >"${stats_lock_dir}/pid"
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
"status",
|
||||
"config",
|
||||
"statistics",
|
||||
"statistics_summary",
|
||||
"rules_list"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -24,6 +24,10 @@ return {
|
||||
call: statistics.build
|
||||
},
|
||||
|
||||
statistics_summary: {
|
||||
call: statistics.summary
|
||||
},
|
||||
|
||||
config_update: {
|
||||
args: {
|
||||
values: {}
|
||||
|
||||
@@ -84,7 +84,7 @@ function sanitize_source(item) {
|
||||
};
|
||||
}
|
||||
|
||||
function build_statistics() {
|
||||
function build_statistics(include_detail) {
|
||||
reload_uci();
|
||||
let enabled = to_bool(cfg('statistics_enabled', '1'), true);
|
||||
let retention_hours = to_int(cfg('statistics_retention_hours', '168'), 168);
|
||||
@@ -98,7 +98,7 @@ function build_statistics() {
|
||||
if (!effective_enabled) {
|
||||
source.available = false;
|
||||
}
|
||||
return {
|
||||
let result = {
|
||||
schema: {
|
||||
name: STATISTICS_SCHEMA_NAME,
|
||||
version: STATISTICS_SCHEMA_VERSION
|
||||
@@ -133,13 +133,26 @@ function build_statistics() {
|
||||
totals: {
|
||||
queries: to_int(totals.queries, 0),
|
||||
blocked: to_int(totals.blocked, 0)
|
||||
},
|
||||
hourly: sanitize_hourly(data.hourly),
|
||||
device_limit: to_int(data.device_limit, 128),
|
||||
devices_truncated: to_bool(data.devices_truncated, false),
|
||||
devices: sanitize_devices(data.devices)
|
||||
}
|
||||
};
|
||||
// Preserve the legacy detailed RPC field insertion order as well as
|
||||
// names/values: totals -> hourly -> device_limit -> devices_truncated ->
|
||||
// devices. Summary callers omit only the two large detail arrays.
|
||||
if (include_detail !== false) {
|
||||
result.hourly = sanitize_hourly(data.hourly);
|
||||
}
|
||||
result.device_limit = to_int(data.device_limit, 128);
|
||||
result.devices_truncated = to_bool(data.devices_truncated, false);
|
||||
if (include_detail !== false) {
|
||||
result.devices = sanitize_devices(data.devices);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function build_statistics_summary() {
|
||||
return build_statistics(false);
|
||||
}
|
||||
return {
|
||||
build: build_statistics
|
||||
build: build_statistics,
|
||||
summary: build_statistics_summary
|
||||
};
|
||||
@@ -8,6 +8,27 @@ Describe 'SafeShield blocklist behavior'
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'builds large lists with bounded merge scratch files and preserves precedence'
|
||||
When call ss_case_blocklist_tmpfs
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'preserves the exact legacy domain order for both dnsmasq modes'
|
||||
When call ss_case_blocklist_legacy_order
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'filters block rules with bounded-memory lookup and no-allow fast path'
|
||||
When call ss_case_blocklist_low_memory
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'resolves and downloads multiple Hub artifact sources'
|
||||
When call ss_case_multi_artifact
|
||||
The status should be success
|
||||
|
||||
@@ -50,6 +50,13 @@ Describe 'SafeShield core shell behavior'
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'updates status fields in batches rather than one process per key'
|
||||
When call ss_case_status_batch_writes
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'removes preserved legacy license keys during upgrade'
|
||||
When call ss_case_legacy_license_cleanup
|
||||
The status should be success
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# shellcheck shell=sh
|
||||
|
||||
Describe 'SafeShield lifecycle and refresh state machines'
|
||||
It 'rolls back full refresh failures while preserving the active blocklist for upgrade-required responses'
|
||||
It 'keeps DNS running before installation failures and rolls back post-install failures'
|
||||
When call ss_case_refresh_state_machine
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# shellcheck shell=sh
|
||||
|
||||
Describe 'SafeShield directory locking and state consistency'
|
||||
It 'prevents lock theft, reclaims stale owners and respects ownership'
|
||||
When call ss_case_lock_owner_and_contention
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'serializes concurrent operations using BusyBox-compatible locks'
|
||||
When call ss_case_lock_parallel
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'serializes status read-modify-write operations under the same lock'
|
||||
When call ss_case_status_write_serialization
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'does not corrupt the previous status when serialization fails'
|
||||
When call ss_case_status_write_failure
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
End
|
||||
@@ -16,6 +16,8 @@ export SS_SPEC_ROOT
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/spec/support/lifecycle_cases.sh"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/spec/support/locking_cases.sh"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/spec/support/blocklist_cases.sh"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/spec/support/device_api_cases.sh"
|
||||
|
||||
@@ -119,6 +119,13 @@ Describe 'SafeShield statistics behavior'
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'prunes expired statistics while preserving the short upload window'
|
||||
When call ss_case_statistics_retention_cache
|
||||
The status should be success
|
||||
The output should equal ''
|
||||
The error should equal ''
|
||||
End
|
||||
|
||||
It 'loads the statistics collector from focused awk modules'
|
||||
When call ss_case_statistics_modules
|
||||
The status should be success
|
||||
|
||||
@@ -140,6 +140,203 @@ sample-two.example'
|
||||
ss_spec_assert_eq "$DNS_CHECK_COUNT" '2'
|
||||
)
|
||||
|
||||
# A larger, overlapping fixture exercises sorted streaming, tmpfs cleanup and
|
||||
# precedence without downloading any external artifact.
|
||||
ss_case_blocklist_tmpfs() (
|
||||
set -eu
|
||||
TMP_DIR="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
|
||||
SS_TMP_DIR="$TMP_DIR/tmp"
|
||||
SS_DNSMASQ_DIR="$TMP_DIR/dnsmasq.d"
|
||||
SS_BLOCKLIST_FILE="$SS_DNSMASQ_DIR/safeshield.blocklist"
|
||||
ss_min_valid_line_count=1
|
||||
ss_max_blocklist_file_size_kb=1024
|
||||
ss_dnsmasq_sanity_check=0
|
||||
mkdir -p "$SS_TMP_DIR" "$SS_DNSMASQ_DIR"
|
||||
ss_should_stop() { [ "${STOP_NOW:-0}" = 1 ]; }
|
||||
ss_status_set() { :; }
|
||||
ss_status_add_error() { :; }
|
||||
log_info() { :; }
|
||||
log_error() { :; }
|
||||
ss_blocklist_tmp_path() { printf '%s/.safeshield.blocklist.tmp.%s\n' "$SS_DNSMASQ_DIR" "$$"; }
|
||||
ss_install_blocklist_atomic() { mv -f "$1" "$2"; }
|
||||
DNSMASQ_PATCHED=1
|
||||
ss_dnsmasq_safeshield_block_supported() { [ "$DNSMASQ_PATCHED" = '1' ]; }
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
|
||||
|
||||
awk 'BEGIN { for (i = 1; i <= 6000; i++) printf "ads%05d.example\n", i }' >"$SS_TMP_DIR/api.0.block.txt"
|
||||
awk 'BEGIN { for (i = 4000; i <= 10000; i++) printf "ads%05d.example\n", i }' >"$SS_TMP_DIR/api.1.block.txt"
|
||||
printf '%s\n' 'ads00042.example' 'hub-allow.example' >"$SS_TMP_DIR/api.2.allow.txt"
|
||||
printf '%s\n' 'hub-allow.example' 'local.example' >"$SS_TMP_DIR/local.block.txt"
|
||||
printf '%s\n' 'local.example' 'ads00042.example' >"$SS_TMP_DIR/local.allow.txt"
|
||||
|
||||
# Prior implementations left 7 merge scratch files on tmpfs. Even stale
|
||||
# files from a previous run must be released after this refresh.
|
||||
printf 'stale\n' >"$SS_TMP_DIR/effective-api-blocks.domains"
|
||||
printf 'stale\n' >"$SS_TMP_DIR/final-blocks.domains"
|
||||
ss_merge_lists
|
||||
ss_spec_assert_eq "$(grep -c '^safeshield-block=/' "$SS_BLOCKLIST_FILE")" '10000'
|
||||
ss_spec_assert_eq "$(grep -c '^server=/' "$SS_BLOCKLIST_FILE")" '2'
|
||||
ss_spec_assert_file_contains "$SS_BLOCKLIST_FILE" 'safeshield-block=/hub-allow.example/'
|
||||
ss_spec_assert_file_not_contains "$SS_BLOCKLIST_FILE" 'safeshield-block=/ads00042.example/'
|
||||
ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'server=/ads00042.example/#'
|
||||
ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'server=/local.example/#'
|
||||
# Cached source shards and local rules must not be deleted by cleanup.
|
||||
ss_spec_assert_exists "$SS_TMP_DIR/api.0.block.txt"
|
||||
ss_spec_assert_exists "$SS_TMP_DIR/local.block.txt"
|
||||
for derived in \
|
||||
api-blocks.domains api-allows.domains \
|
||||
effective-local-blocks.domains effective-api-allows.domains \
|
||||
effective-allows.domains effective-api-blocks.domains final-blocks.domains; do
|
||||
ss_spec_assert_not_exists "$SS_TMP_DIR/$derived"
|
||||
done
|
||||
|
||||
# Stock dnsmasq rules retain identical precedence and order.
|
||||
DNSMASQ_PATCHED=0
|
||||
ss_merge_lists
|
||||
ss_spec_assert_eq "$(grep -c '^address=/' "$SS_BLOCKLIST_FILE")" '10000'
|
||||
ss_spec_assert_file_line "$SS_BLOCKLIST_FILE" 'address=/hub-allow.example/#'
|
||||
ss_spec_assert_file_not_contains "$SS_BLOCKLIST_FILE" 'address=/ads00042.example/#'
|
||||
for derived in api-blocks.domains effective-api-blocks.domains final-blocks.domains; do
|
||||
ss_spec_assert_not_exists "$SS_TMP_DIR/$derived"
|
||||
done
|
||||
|
||||
# A cancelled refresh must not install partial rules or retain scratch.
|
||||
before="$(cksum "$SS_BLOCKLIST_FILE")"
|
||||
STOP_NOW=1
|
||||
result=0
|
||||
ss_merge_lists || result=$?
|
||||
ss_spec_assert_eq "$result" '130'
|
||||
ss_spec_assert_eq "$(cksum "$SS_BLOCKLIST_FILE")" "$before"
|
||||
ss_spec_assert_not_exists "$SS_TMP_DIR/api-blocks.domains"
|
||||
ss_spec_assert_not_exists "$SS_TMP_DIR/effective-api-blocks.domains"
|
||||
)
|
||||
|
||||
# Print an actionable diff if this ordering contract regresses in CI.
|
||||
ss_assert_legacy_blocklist_order() (
|
||||
# Use a subshell to keep these variables scoped under POSIX sh.
|
||||
expected="$1"
|
||||
actual_file="$2"
|
||||
expected_file="${actual_file}.expected.$$"
|
||||
|
||||
printf '%s\n' "$expected" >"$expected_file"
|
||||
if ! cmp -s "$expected_file" "$actual_file"; then
|
||||
diff -u "$expected_file" "$actual_file" >&2 || :
|
||||
rm -f "$expected_file"
|
||||
return 1
|
||||
fi
|
||||
rm -f "$expected_file"
|
||||
)
|
||||
|
||||
# Ordering is part of our byte-for-byte legacy blocklist compatibility.
|
||||
# Sorting decorated dnsmasq rule strings changes prefix ordering ("foo" vs
|
||||
# "foo-bar" / "foo.bar"); always compare by the undressed domain field.
|
||||
ss_case_blocklist_legacy_order() (
|
||||
set -eu
|
||||
TMP_DIR="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
|
||||
SS_TMP_DIR="$TMP_DIR/tmp"
|
||||
SS_DNSMASQ_DIR="$TMP_DIR/dnsmasq.d"
|
||||
SS_BLOCKLIST_FILE="$SS_DNSMASQ_DIR/safeshield.blocklist"
|
||||
ss_min_valid_line_count=0
|
||||
ss_max_blocklist_file_size_kb=1024
|
||||
ss_dnsmasq_sanity_check=0
|
||||
mkdir -p "$SS_TMP_DIR" "$SS_DNSMASQ_DIR"
|
||||
ss_should_stop() { return 1; }
|
||||
ss_status_set() { :; }
|
||||
ss_status_add_error() { :; }
|
||||
log_info() { :; }
|
||||
log_error() { :; }
|
||||
ss_blocklist_tmp_path() { printf '%s/.safeshield.blocklist.tmp.%s\n' "$SS_DNSMASQ_DIR" "$$"; }
|
||||
ss_install_blocklist_atomic() { mv -f "$1" "$2"; }
|
||||
DNSMASQ_PATCHED=1
|
||||
ss_dnsmasq_safeshield_block_supported() { [ "$DNSMASQ_PATCHED" = '1' ]; }
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
|
||||
# API shards are pre-sorted by ss_download_api_artifacts(). Keep this
|
||||
# fixture representative of the actual -m (merge) precondition.
|
||||
printf '%s\n' 'abc.test' 'foo' 'foo-bar' 'foo.bar' 'foo_bar' |
|
||||
LC_ALL=C sort -u >"$SS_TMP_DIR/api.0.block.txt"
|
||||
printf '%s\n' 'abc.test-more' 'abc.test.uk' |
|
||||
LC_ALL=C sort -u >"$SS_TMP_DIR/api.1.block.txt"
|
||||
ss_merge_lists
|
||||
expected='safeshield-block=/abc.test/
|
||||
safeshield-block=/abc.test-more/
|
||||
safeshield-block=/abc.test.uk/
|
||||
safeshield-block=/foo/
|
||||
safeshield-block=/foo-bar/
|
||||
safeshield-block=/foo.bar/
|
||||
safeshield-block=/foo_bar/'
|
||||
ss_assert_legacy_blocklist_order "$expected" "$SS_BLOCKLIST_FILE"
|
||||
DNSMASQ_PATCHED=0
|
||||
ss_merge_lists
|
||||
expected='address=/abc.test/#
|
||||
address=/abc.test-more/#
|
||||
address=/abc.test.uk/#
|
||||
address=/foo/#
|
||||
address=/foo-bar/#
|
||||
address=/foo.bar/#
|
||||
address=/foo_bar/#'
|
||||
ss_assert_legacy_blocklist_order "$expected" "$SS_BLOCKLIST_FILE"
|
||||
)
|
||||
|
||||
# Both low-memory filtering paths must retain the legacy allow precedence and
|
||||
# the original local-block-first rule stream. The final merge sorts afterwards.
|
||||
ss_case_blocklist_low_memory() (
|
||||
set -eu
|
||||
TMP_DIR="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
|
||||
mkdir -p "$TMP_DIR"
|
||||
SS_TMP_DIR="$TMP_DIR"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
|
||||
cat >"$TMP_DIR/blocks" <<'DATA'
|
||||
ads.example
|
||||
foo
|
||||
foo-bar
|
||||
sub.ads.example
|
||||
invalid/path
|
||||
CAPS.example
|
||||
valid.example
|
||||
DATA
|
||||
printf '%s\n' 'local.example' >"$TMP_DIR/local"
|
||||
: >"$TMP_DIR/shadow"
|
||||
ss_write_effective_block_rules \
|
||||
"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
|
||||
"$TMP_DIR/result" 'safeshield-block=/' '/'
|
||||
expected='safeshield-block=/local.example/
|
||||
safeshield-block=/ads.example/
|
||||
safeshield-block=/foo/
|
||||
safeshield-block=/foo-bar/
|
||||
safeshield-block=/sub.ads.example/
|
||||
safeshield-block=/valid.example/'
|
||||
ss_spec_assert_eq "$(cat "$TMP_DIR/result")" "$expected"
|
||||
|
||||
# A present allow shadows the matching Hub block and its descendants, but
|
||||
# does not remove an explicit local block or unrelated Hub domains.
|
||||
printf '%s\n' 'foo' 'ads.example' >"$TMP_DIR/shadow"
|
||||
ss_write_effective_block_rules \
|
||||
"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
|
||||
"$TMP_DIR/result" 'address=/' '/#'
|
||||
expected='address=/local.example/#
|
||||
address=/foo-bar/#
|
||||
address=/valid.example/#'
|
||||
ss_spec_assert_eq "$(cat "$TMP_DIR/result")" "$expected"
|
||||
|
||||
# When the local list is empty, the no-allow fast path must still process
|
||||
# Hub block records (including the first record of the second input file).
|
||||
: >"$TMP_DIR/local"
|
||||
: >"$TMP_DIR/shadow"
|
||||
ss_write_effective_block_rules \
|
||||
"$TMP_DIR/shadow" "$TMP_DIR/local" "$TMP_DIR/blocks" \
|
||||
"$TMP_DIR/result" 'address=/' '/#'
|
||||
ss_spec_assert_eq "$(head -n 1 "$TMP_DIR/result")" 'address=/ads.example/#'
|
||||
ss_spec_assert_eq "$(grep -c '^address=/' "$TMP_DIR/result")" '5'
|
||||
BLOCKLIST="$SS_SPEC_ROOT/files/usr/lib/safeshield/blocklist.sh"
|
||||
ss_spec_assert_file_contains "$BLOCKLIST" 'sort -S 1M -u -t/ -k2,2 -o "$final" "$final"'
|
||||
)
|
||||
|
||||
ss_case_multi_artifact() (
|
||||
set -eu
|
||||
TMP_DIR="$(ss_spec_tmpdir)"
|
||||
|
||||
@@ -6,6 +6,7 @@ ss_case_utils() (
|
||||
PKG_NAME='safeshield-test'
|
||||
export PKG_NAME
|
||||
# shellcheck disable=SC1091
|
||||
SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
|
||||
|
||||
command_exists sh
|
||||
@@ -25,6 +26,7 @@ ss_case_config_validation() (
|
||||
PKG_NAME='safeshield-test'
|
||||
export PKG_NAME
|
||||
# shellcheck disable=SC1091
|
||||
SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
|
||||
|
||||
WARNINGS=''
|
||||
@@ -121,6 +123,7 @@ ss_case_dnsmasq_version() (
|
||||
PKG_NAME='safeshield'
|
||||
export PKG_NAME
|
||||
# shellcheck disable=SC1091
|
||||
SS_LOCKING_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/utils.sh"
|
||||
DNSMASQ_FEATURE_HEALTH=''
|
||||
ss_status_set() {
|
||||
@@ -378,6 +381,15 @@ ss_case_status_state() (
|
||||
record() { printf '%s\t%s\t%s\n' "$1" "${2:-}" "${3:-}" >>"$CALLS"; }
|
||||
ss_status_set() { record set "$1" "$2"; }
|
||||
ss_status_add_error() { record error "$1"; }
|
||||
# Capture each value for existing state assertions while recording that a
|
||||
# whole group is now committed through one locked status-store operation.
|
||||
ss_status_set_many() {
|
||||
record batch set_many "$#"
|
||||
while [ "$#" -ge 2 ]; do
|
||||
record set "$1" "$2"
|
||||
shift 2
|
||||
done
|
||||
}
|
||||
ss_status_apply() {
|
||||
record apply "$1" "${2:-}"
|
||||
return 0
|
||||
@@ -388,6 +400,7 @@ ss_case_status_state() (
|
||||
[ "$1" -ge 0 ] 2>/dev/null
|
||||
}
|
||||
ss_status_mark_failure 'artifact_download_failed'
|
||||
ss_spec_assert_eq "$(grep -c "$(printf 'batch\tset_many')" "$CALLS")" '1'
|
||||
grep -F "$(printf 'set\tstatus\terror')" "$CALLS" >/dev/null
|
||||
grep -F "$(printf 'set\tlast_failure\t1000')" "$CALLS" >/dev/null
|
||||
grep -F "$(printf 'set\tlast_result\terror')" "$CALLS" >/dev/null
|
||||
@@ -405,6 +418,7 @@ ss_case_status_state() (
|
||||
! grep -F "$(printf 'set\tnext_refresh_at')" "$CALLS" >/dev/null
|
||||
: >"$CALLS"
|
||||
ss_status_reset_health_fields
|
||||
ss_spec_assert_eq "$(grep -c "$(printf 'batch\tset_many')" "$CALLS")" '1'
|
||||
for key in health_dnsmasq_binary health_dns_runtime health_artifact_download dnsmasq_min_version; do
|
||||
grep -F "$(printf 'set\t%s\t' "$key")" "$CALLS" >/dev/null
|
||||
done
|
||||
@@ -467,8 +481,45 @@ ss_case_status_version() (
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/Makefile" "echo '\$(PKG_VERSION)-r\$(PKG_RELEASE)' > \$(1)/usr/lib/safeshield/version"
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/share/rpcd/ucode/safeshield/core.uc" 'fs.readfile(`/usr/lib/${PKG_NAME}/version`)'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/share/rpcd/ucode/safeshield/status.uc" 'version: PKG_VERSION,'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'safeshield_upgrade_required'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'failure_code="$(ss_resolve_error_code)"'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'failure_code="$(ss_artifact_download_error_code)"'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'if [ "$failure_code" != "safeshield_upgrade_required" ]; then'
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/files/usr/lib/safeshield/core.sh" 'No blocklist replacement has happened'
|
||||
)
|
||||
|
||||
# The legacy refresh reset used one ucode process + one locked JSON rewrite
|
||||
# per key. Grouping related updates keeps the same values with fewer writes.
|
||||
ss_case_status_batch_writes() (
|
||||
set -eu
|
||||
TMP_DIR="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT HUP INT TERM
|
||||
RUNNING_STATUS_FILE="$TMP_DIR/status.json"
|
||||
# shellcheck disable=SC1091
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
|
||||
CALLS="$TMP_DIR/calls"
|
||||
: >"$CALLS"
|
||||
ss_status_apply() {
|
||||
printf '%s\n' "$1" >>"$CALLS"
|
||||
return 0
|
||||
}
|
||||
ss_now() { printf '%s\n' 1000; }
|
||||
is_valid_integer() {
|
||||
case "$1" in '' | *[!0-9]*) return 1 ;; esac
|
||||
return 0
|
||||
}
|
||||
|
||||
ss_status_reset
|
||||
ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '6'
|
||||
ss_spec_assert_eq "$(grep -c '^set_many$' "$CALLS")" '4'
|
||||
ss_spec_assert_eq "$(grep -c '^reset$' "$CALLS")" '1'
|
||||
ss_spec_assert_eq "$(grep -c '^clear_messages$' "$CALLS")" '1'
|
||||
|
||||
: >"$CALLS"
|
||||
ss_status_mark_success 300
|
||||
ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '2'
|
||||
ss_spec_assert_file_line "$CALLS" 'set_many'
|
||||
|
||||
: >"$CALLS"
|
||||
ss_status_mark_failure artifact_download_failed
|
||||
ss_spec_assert_eq "$(wc -l <"$CALLS" | tr -d '[:space:]')" '2'
|
||||
ss_spec_assert_file_line "$CALLS" 'add_error'
|
||||
)
|
||||
@@ -138,6 +138,7 @@ ss_case_refresh_state_machine() (
|
||||
ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
|
||||
else
|
||||
! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
|
||||
! grep -F 'dnsmasq_restart ' "$CALLS" >/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
@@ -157,7 +158,10 @@ ss_case_refresh_state_machine() (
|
||||
ss_spec_assert_file_line "$CALLS" 'lock_close'
|
||||
! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
|
||||
|
||||
assert_refresh_failure merge merge_failed 1
|
||||
assert_refresh_failure merge merge_failed 0
|
||||
assert_refresh_failure resolve artifact_resolve_failed 0
|
||||
assert_refresh_failure download artifact_download_failed 0
|
||||
assert_refresh_failure local_rules allowlist_build_failed 0
|
||||
assert_refresh_failure install install_failed 1
|
||||
assert_refresh_failure restart dnsmasq_restart_failed 1
|
||||
assert_refresh_failure runtime dns_runtime_check_failed 1
|
||||
@@ -313,7 +317,12 @@ ss_case_local_apply_state_machine() (
|
||||
verify) code='local_blocklist_verification_failed' ;;
|
||||
esac
|
||||
ss_spec_assert_file_line "$CALLS" "local_failure $code"
|
||||
ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
|
||||
if [ "$scenario" = 'merge' ]; then
|
||||
! grep -Fx 'restore_and_restart' "$CALLS" >/dev/null
|
||||
! grep -Fx 'dnsmasq_restart' "$CALLS" >/dev/null
|
||||
else
|
||||
ss_spec_assert_file_line "$CALLS" 'restore_and_restart'
|
||||
fi
|
||||
ss_spec_assert_file_line "$CALLS" 'lock_close'
|
||||
done
|
||||
|
||||
@@ -333,54 +342,42 @@ ss_case_refresh_locking() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
RUNNING_STATUS_FILE="$TMP/status.json"
|
||||
SS_REFRESH_LOCK="$TMP/safeshield-refresh.lock"
|
||||
SS_REFRESH_LOCK_FD=9
|
||||
MOCK_FLOCK_DIR="$TMP/mock-flock.lock"
|
||||
|
||||
# Keep this regression test portable on development hosts such as macOS,
|
||||
# where flock(1) is not installed by default. The production code still
|
||||
# invokes flock -n/-u; this shim emulates those operations with atomic
|
||||
# mkdir/rmdir so contention, release, timeout, and termination paths are
|
||||
# exercised without depending on a host-specific flock binary.
|
||||
flock() {
|
||||
case "${1:-}" in
|
||||
-n)
|
||||
mkdir "$MOCK_FLOCK_DIR" 2>/dev/null
|
||||
;;
|
||||
-u)
|
||||
rmdir "$MOCK_FLOCK_DIR" 2>/dev/null || true
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
return 2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# shellcheck disable=SC1091
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
|
||||
is_valid_integer() { case "$1" in '' | *[!0-9]*) return 1 ;; *) return 0 ;; esac }
|
||||
ss_should_stop() { return 1; }
|
||||
|
||||
mkdir "$MOCK_FLOCK_DIR"
|
||||
ss_refresh_lock_open
|
||||
ss_spec_assert_eq "$(cat "${SS_REFRESH_LOCK}.d/pid")" "$$"
|
||||
# A separate shell must not acquire or release our live lock.
|
||||
! sh -c '. "$1"; ss_lock_try "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "${SS_REFRESH_LOCK}.d"
|
||||
ss_refresh_lock_close
|
||||
ss_spec_assert_not_exists "${SS_REFRESH_LOCK}.d"
|
||||
|
||||
# Simulate an owner which crashed after writing pid.
|
||||
mkdir "${SS_REFRESH_LOCK}.d"
|
||||
printf '99999999\n' >"${SS_REFRESH_LOCK}.d/pid"
|
||||
ss_refresh_lock_open
|
||||
ss_refresh_lock_close
|
||||
|
||||
# A freshly-created empty lock must not be stolen while the owner writes pid.
|
||||
mkdir "${SS_REFRESH_LOCK}.d"
|
||||
! ss_refresh_lock_open
|
||||
! ss_refresh_lock_open_wait 0
|
||||
rmdir "$MOCK_FLOCK_DIR"
|
||||
rmdir "${SS_REFRESH_LOCK}.d"
|
||||
|
||||
ss_refresh_lock_open
|
||||
ss_spec_assert_exists "$MOCK_FLOCK_DIR"
|
||||
ss_refresh_lock_close
|
||||
ss_spec_assert_not_exists "$MOCK_FLOCK_DIR"
|
||||
|
||||
mkdir "$MOCK_FLOCK_DIR"
|
||||
ss_should_stop() { return 0; }
|
||||
mkdir "${SS_REFRESH_LOCK}.d"
|
||||
printf '%s\n' "$$" >"${SS_REFRESH_LOCK}.d/pid"
|
||||
set +e
|
||||
ss_refresh_lock_open_wait 10
|
||||
rc=$?
|
||||
set -e
|
||||
rmdir "$MOCK_FLOCK_DIR"
|
||||
ss_spec_assert_eq "$rc" '130'
|
||||
ss_refresh_lock_close
|
||||
)
|
||||
|
||||
ss_case_service_lifecycle() (
|
||||
|
||||
Executable
+114
@@ -0,0 +1,114 @@
|
||||
#!/bin/sh
|
||||
# shellcheck shell=sh
|
||||
|
||||
ss_case_lock_owner_and_contention() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
LOCK="$TMP/collector.lock"
|
||||
ss_lock_try "$LOCK"
|
||||
[ "$(cat "$LOCK/pid")" = "$$" ]
|
||||
# Different process may not acquire the lock or remove its owner.
|
||||
! sh -c '. "$1"; ss_lock_try "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "$LOCK"
|
||||
! sh -c '. "$1"; ss_lock_release "$2"' sh "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" "$LOCK"
|
||||
[ "$(cat "$LOCK/pid")" = "$$" ]
|
||||
ss_lock_release "$LOCK"
|
||||
[ ! -e "$LOCK" ]
|
||||
# Stale pid may be reclaimed by exactly one contender.
|
||||
mkdir "$LOCK"
|
||||
printf '99999999\n' >"$LOCK/pid"
|
||||
ss_lock_try "$LOCK"
|
||||
[ "$(cat "$LOCK/pid")" = "$$" ]
|
||||
ss_lock_release "$LOCK"
|
||||
# Do not seize a newly created lock before its pid has been populated.
|
||||
mkdir "$LOCK"
|
||||
! ss_lock_try "$LOCK"
|
||||
[ -d "$LOCK" ]
|
||||
rmdir "$LOCK"
|
||||
# An abandoned pid-less directory (including a crashed recovery guard)
|
||||
# becomes reclaimable after its initialization grace period.
|
||||
mkdir "$LOCK" "${LOCK}.reclaim"
|
||||
touch -t 202001010000 "$LOCK" "${LOCK}.reclaim"
|
||||
ss_lock_try "$LOCK"
|
||||
ss_lock_release "$LOCK"
|
||||
)
|
||||
|
||||
ss_case_lock_parallel() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
LOCK="$TMP/write.lock"
|
||||
RESULT="$TMP/results"
|
||||
for id in 1 2 3 4 5 6 7 8; do
|
||||
sh -c '
|
||||
. "$1"
|
||||
ss_lock_wait "$2" 15 || exit 1
|
||||
printf "%s\n" "$3" >>"$4"
|
||||
sleep 0.05
|
||||
ss_lock_release "$2"
|
||||
' sh "$LIB" "$LOCK" "$id" "$RESULT" &
|
||||
done
|
||||
wait
|
||||
[ "$(wc -l <"$RESULT")" -eq 8 ]
|
||||
[ ! -e "$LOCK" ]
|
||||
[ ! -e "${LOCK}.reclaim" ]
|
||||
)
|
||||
|
||||
ss_case_status_write_serialization() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
STATUS="$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
|
||||
# Test the real shell writer with a slow stand-in for the ucode serializer.
|
||||
# A serializer must observe all previously committed updates under the same lock.
|
||||
cat >"$TMP/writer.sh" <<'EOF_WRITER'
|
||||
#!/bin/sh
|
||||
. "$1"
|
||||
. "$2"
|
||||
RUNNING_STATUS_FILE="$3"
|
||||
RUNNING_STATUS_LOCK="$4"
|
||||
json_lock_open() { ss_lock_wait "$RUNNING_STATUS_LOCK" 10; }
|
||||
json_lock_close() { ss_lock_release "$RUNNING_STATUS_LOCK"; }
|
||||
ss_status_exec() {
|
||||
key="$2"
|
||||
[ -f "$RUNNING_STATUS_FILE" ] && cat "$RUNNING_STATUS_FILE"
|
||||
sleep 0.05
|
||||
printf '%s\n' "$key"
|
||||
}
|
||||
ss_status_apply set "$5" unused
|
||||
EOF_WRITER
|
||||
for n in 1 2 3 4 5 6 7 8; do
|
||||
sh "$TMP/writer.sh" "$LIB" "$STATUS" "$TMP/status.json" "$TMP/status.lock" "$n" &
|
||||
done
|
||||
wait
|
||||
[ "$(wc -l <"$TMP/status.json")" -eq 8 ]
|
||||
[ ! -e "$TMP/status.lock" ]
|
||||
[ ! -e "$TMP/status.json.tmp" ]
|
||||
)
|
||||
|
||||
ss_case_status_write_failure() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh"
|
||||
# shellcheck disable=SC1090
|
||||
. "$SS_SPEC_ROOT/files/usr/lib/safeshield/status.sh"
|
||||
RUNNING_STATUS_LOCK="$TMP/status.lock"
|
||||
RUNNING_STATUS_FILE="$TMP/status.json"
|
||||
json_lock_open() { ss_lock_wait "$RUNNING_STATUS_LOCK" 10; }
|
||||
json_lock_close() { ss_lock_release "$RUNNING_STATUS_LOCK"; }
|
||||
printf 'original\n' >"$RUNNING_STATUS_FILE"
|
||||
ss_status_exec() {
|
||||
printf 'partial\n'
|
||||
return 1
|
||||
}
|
||||
! ss_status_apply set key value
|
||||
[ "$(cat "$RUNNING_STATUS_FILE")" = 'original' ]
|
||||
[ ! -e "${RUNNING_STATUS_FILE}.tmp.$$" ]
|
||||
[ ! -e "$RUNNING_STATUS_LOCK" ]
|
||||
)
|
||||
@@ -115,6 +115,7 @@ EOF_IP
|
||||
PATH="$TMP/bin:$PATH" \
|
||||
SS_STATSD_FUNCTIONS_LIB="$TMP/functions.sh" \
|
||||
SS_STATSD_CORE_LIB="$TMP/core.sh" \
|
||||
SS_STATSD_LOCK_LIB="$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" \
|
||||
SS_STATSD_AWK_DIR="$SS_SPEC_ROOT/files/usr/lib/safeshield/statistics" \
|
||||
SS_STATSD_POLL_COMMAND="$TMP/bin/stats-poll" \
|
||||
SS_STATSD_GENERATION_ID='runtime-generation' \
|
||||
@@ -580,6 +581,7 @@ EOF_CORE
|
||||
write_payload "$SS_STATISTICS_UPLOAD_JSON_FILE" 7
|
||||
upload_pending_create
|
||||
ss_spec_assert_file_contains "$SS_STATISTICS_UPLOAD_PENDING_META_FILE" "$(printf 'recent\tgeneration-upload\t7')"
|
||||
ss_should_terminate=0
|
||||
MOCK_HTTP_MODE='stale'
|
||||
export MOCK_HTTP_MODE
|
||||
upload_send_pending
|
||||
|
||||
@@ -1272,3 +1272,37 @@ DATA
|
||||
! grep -F '"bucket_start":1787950800' "$UPLOAD" >/dev/null
|
||||
ss_spec_assert_eq "$(sed -n 's/.*"snapshot_seq":\([0-9][0-9]*\).*/\1/p' "$UPLOAD")" "$(sed -n 's/.*"snapshot_seq":\([0-9][0-9]*\).*/\1/p' "$JSON")"
|
||||
)
|
||||
|
||||
# Verifies that caching the retention sweep never leaves expired history in
|
||||
# the local JSON, the persisted state, or the short cloud upload projection.
|
||||
ss_case_statistics_retention_cache() (
|
||||
set -eu
|
||||
TMP="$(ss_spec_tmpdir)"
|
||||
trap 'rm -rf "$TMP"' EXIT HUP INT TERM
|
||||
STATE="$TMP/state.tsv"
|
||||
JSON="$TMP/statistics.json"
|
||||
UPLOAD="$TMP/upload.json"
|
||||
INPUT="$TMP/input.log"
|
||||
LEASES="$TMP/dhcp.leases"
|
||||
: >"$LEASES"
|
||||
cat >"$INPUT" <<'DATA'
|
||||
daemon.info dnsmasq[1]: query[A] one.example from 192.168.1.1
|
||||
daemon.info dnsmasq[1]: query[A] two.example from 192.168.1.1
|
||||
daemon.info dnsmasq[1]: query[A] three.example from 192.168.1.1
|
||||
DATA
|
||||
ss_statistics_awk \
|
||||
-v state_file="$STATE" -v json_file="$JSON" \
|
||||
-v upload_json_file="$UPLOAD" -v lease_file="$LEASES" \
|
||||
-v snapshot_interval=1 -v retention_hours=2 \
|
||||
-v upload_window_hours=1 -v provisional_retention_hours=2 \
|
||||
-v generation_seed='retention-test' \
|
||||
-v fixed_now=1787950800 -v fixed_step=3600 <"$INPUT"
|
||||
|
||||
# The final event is in the third hour. The two-hour retention window
|
||||
# contains exactly the second and third events.
|
||||
ss_spec_assert_file_contains "$JSON" '"totals":{"queries":2,"blocked":0}'
|
||||
ss_spec_assert_file_contains "$JSON" '"retention_hours":2'
|
||||
ss_spec_assert_file_contains "$UPLOAD" '"totals":{"queries":1,"blocked":0}'
|
||||
! grep -F 'bucket' "$STATE" | grep -F '1787950800' >/dev/null
|
||||
ss_spec_assert_file_contains "$STATE" "$(printf 'bucket\t1787958000\t1\t0')"
|
||||
)
|
||||
@@ -12,6 +12,8 @@ ss_case_tooling() (
|
||||
[ -f "$SS_SPEC_ROOT/spec/statistics_spec.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/spec/ucode_spec.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/spec/tooling_spec.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/spec/locking_spec.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/files/usr/lib/safeshield/locking.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/scripts/lint.sh" ]
|
||||
[ -f "$SS_SPEC_ROOT/.github/workflows/lint-shell.yml" ]
|
||||
ss_spec_assert_file_contains "$SS_SPEC_ROOT/.pre-commit-config.yaml" 'entry: sh scripts/lint.sh'
|
||||
|
||||
@@ -48,6 +48,21 @@ ss_case_ucode() (
|
||||
out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" set stage running)"
|
||||
printf '%s\n' "$out" >"$status_file"
|
||||
printf '%s\n' "$out" | grep -F '"stage":"running"' >/dev/null
|
||||
# One invocation must update every pair without dropping existing status.
|
||||
out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many status ready stage 'done' count 0 optional '')"
|
||||
printf '%s\n' "$out" >"$status_file"
|
||||
printf '%s\n' "$out" | grep -F '"status":"ready"' >/dev/null
|
||||
printf '%s\n' "$out" | grep -F '"stage":"done"' >/dev/null
|
||||
printf '%s\n' "$out" | grep -F '"count":"0"' >/dev/null
|
||||
printf '%s\n' "$out" | grep -F '"optional":""' >/dev/null
|
||||
printf '%s\n' "$out" | grep -F '"stage":"running"' >/dev/null && return 1
|
||||
# Reject malformed pair lists, never serialize partial changes.
|
||||
if "$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many incomplete >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
if "$UCODE_BIN" "$STATUS_STORE" "$status_file" set_many '' value >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
out="$("$UCODE_BIN" "$STATUS_STORE" "$status_file" add_warning dnsmasq_warning)"
|
||||
printf '%s\n' "$out" >"$status_file"
|
||||
printf '%s\n' "$out" | grep -F '"code":"dnsmasq_warning"' >/dev/null
|
||||
|
||||
@@ -28,3 +28,15 @@ assert(result.devices[0].identified == true, 'device identified flag is normaliz
|
||||
assert(result.devices[0].queries == 50 && result.devices[0].blocked == 5, 'device counters are normalized');
|
||||
|
||||
print('ucode statistics tests: ok\n');
|
||||
|
||||
// Summary is deliberately opt-in. It must not duplicate large per-device
|
||||
// arrays while preserving the health/totals metadata used by lightweight UIs.
|
||||
let summary = statistics.summary();
|
||||
assert(summary.schema.name == result.schema.name && summary.schema.version == result.schema.version, 'summary preserves schema');
|
||||
assert(summary.snapshot_seq == result.snapshot_seq && summary.generation_id == result.generation_id, 'summary preserves freshness');
|
||||
assert(summary.enabled == result.enabled && summary.collector_running == result.collector_running, 'summary preserves service status');
|
||||
assert(summary.source.backend == result.source.backend && summary.source.tracked_clients == result.source.tracked_clients, 'summary preserves source health');
|
||||
assert(summary.totals.queries == result.totals.queries && summary.totals.blocked == result.totals.blocked, 'summary preserves totals');
|
||||
assert(summary.device_limit == result.device_limit && summary.devices_truncated == result.devices_truncated, 'summary preserves device capacity metadata');
|
||||
assert(summary.hourly == null && summary.devices == null, 'summary omits detailed arrays');
|
||||
assert(length(result.hourly) == 2 && length(result.devices) == 1, 'full statistics remains unchanged after summary');
|
||||
Reference in new issue
Block a user