mirror of
https://github.com/kiddin9/op-packages.git
synced 2026-11-17 05:53:03 +08:00
412 lines
15 KiB
Bash
Executable File
412 lines
15 KiB
Bash
Executable File
#!/bin/sh
|
|
# dae-geo-update — fetch the newest geoip.dat / geosite.dat for the dae engine
|
|
# and restart dae so it picks them up.
|
|
#
|
|
# Started by the "Update now" button of the LuCI page (its rpcd backend runs
|
|
# this script as a background job and polls the state file) and runnable by
|
|
# hand with /usr/bin/dae-geo-update. There is no cron entry: the update is
|
|
# manual, so a failed download can never take the proxy down unattended.
|
|
#
|
|
# Source:
|
|
# https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@<commit>/geoip.dat
|
|
# https://cdn.jsdelivr.net/gh/Loyalsoldier/v2ray-rules-dat@<commit>/geosite.dat
|
|
# The project publishes no tag (a @<tag> URL returns 404), only the moving
|
|
# "release" branch, so the branch is resolved to its commit first and the files
|
|
# are fetched at that commit. Fetching through the branch is not stable: the
|
|
# branch moves, and jsDelivr keeps a large file in its cache for hours while it
|
|
# revalidates the small sha256 next to it, so a run inside that window gets the
|
|
# previous build and rejects it against the new digest (seen on a live router).
|
|
# The commit URL is immutable, so the two files and their sha256 always come
|
|
# from one snapshot.
|
|
#
|
|
# Target: /usr/share/v2ray — the directory the init script points dae at with
|
|
# DAE_LOCATION_ASSET. dae reads the files when it starts, and only then.
|
|
#
|
|
# State: /var/run/dae-geo-update.state holds "state=running|ok|failed", the pid,
|
|
# the start and finish times and a message token (updated, up_to_date,
|
|
# updated_stopped, download_failed, restart_failed_rollback, rollback_failed).
|
|
# The LuCI backend reads it for the progress display; the state lives on the
|
|
# tmpfs, so a reboot clears it.
|
|
#
|
|
# Log: /var/log/dae-geo-update.log, and every line also goes to syslog under the
|
|
# dae-geo-update tag. /var is a tmpfs on OpenWrt, so the file does not survive a
|
|
# reboot.
|
|
#
|
|
# Backup: /etc/dae/geo-backup holds a copy of each file taken just before this
|
|
# run replaces it. The rollback puts those copies back, so it touches only the
|
|
# files this run changed, never a file an earlier run installed.
|
|
#
|
|
# Download: the request must carry a Range header. Measured on one router, one
|
|
# file, one node: a plain GET ran at 16KB/s, an explicit range request at MB/s.
|
|
# curl's -C - degrades to a plain GET when the local file does not exist yet, so
|
|
# the first attempt always asks for the range explicitly and only a resume of an
|
|
# existing partial file uses -C -.
|
|
#
|
|
# Verification: the sha256 the upstream publishes next to the file is the only
|
|
# trustworthy proof that the download is complete and unmodified — the CDN's
|
|
# content-length and content-range are not (the range total measured smaller
|
|
# than the real file). An unfinished transfer is kept and resumed; a finished
|
|
# file that fails the hash is dropped and fetched from scratch, at most three
|
|
# rounds. When the upstream publishes no sha256 (or it cannot be fetched), one
|
|
# finished transfer is checked against a size range and a first byte of 0x0a,
|
|
# which is a protobuf tag and rules out an HTML error page; a retry cannot
|
|
# improve that check, so it is not retried.
|
|
#
|
|
# Rollback: dae cannot load broken geo data — the configuration fails to build
|
|
# and the proxy stops. After replacing the files the daemon is restarted and
|
|
# checked, and the previous files are put back when it does not come up.
|
|
#
|
|
# Note: the two files under /usr/share/v2ray are installed by the apk packages
|
|
# v2ray-geoip and v2ray-geosite. Upgrading those packages overwrites them with
|
|
# the feed versions; the next run of this script puts the newer files back.
|
|
|
|
set -u
|
|
|
|
PATH=/usr/sbin:/usr/bin:/sbin:/bin
|
|
export PATH
|
|
|
|
DIR=/usr/share/v2ray
|
|
LOGFILE=/var/log/dae-geo-update.log
|
|
STATE_FILE=/var/run/dae-geo-update.state
|
|
BACKUP_DIR=/etc/dae/geo-backup
|
|
WORK=/tmp/dae-geo-update.work
|
|
LOCKFILE=/var/lock/dae-geo-update.lock
|
|
HOST=cdn.jsdelivr.net
|
|
REPO=Loyalsoldier/v2ray-rules-dat
|
|
BRANCH=release
|
|
STARTED="$(date +%s)"
|
|
# Filled by resolve_source() with the commit-pinned or, as a fallback, the
|
|
# branch-based base URL.
|
|
BASE=
|
|
FILES="geosite.dat geoip.dat"
|
|
# A floor for the download, not for the files the packages ship. The two are
|
|
# different things: v2ray-geosite installs a small geosite.dat (a couple of
|
|
# megabytes), while the build this script fetches is ten and more. The floor
|
|
# only ever runs on a file this script downloaded, so it can be set for the
|
|
# latter without rejecting the former.
|
|
MIN_BYTES=5000000
|
|
MAX_BYTES=104857600
|
|
MAX_ROUNDS=3
|
|
FETCH_TIMEOUT=900
|
|
START_TIMEOUT=90
|
|
|
|
log() {
|
|
local line
|
|
line="$(date '+%F %T') $*"
|
|
echo "$line" >> "$LOGFILE" 2>/dev/null
|
|
logger -t dae-geo-update "$*" 2>/dev/null
|
|
}
|
|
|
|
# set_state <state> <message> [finished-at]
|
|
#
|
|
# The file is replaced through a rename so a reader never sees half of it.
|
|
# "finished" stays 0 while the run is in progress.
|
|
set_state() {
|
|
local state="$1" message="$2" finished="${3:-0}"
|
|
|
|
{
|
|
echo "state=$state"
|
|
echo "pid=$$"
|
|
echo "started=$STARTED"
|
|
echo "finished=$finished"
|
|
echo "message=$message"
|
|
} > "$STATE_FILE.new" 2>/dev/null && mv -f "$STATE_FILE.new" "$STATE_FILE" 2>/dev/null
|
|
}
|
|
|
|
# die <message-token> <log text...>
|
|
#
|
|
# The token is what the page shows; the log text is for the operator.
|
|
die() {
|
|
local token="$1"
|
|
shift
|
|
set_state failed "$token" "$(date +%s)"
|
|
log "FATAL: $*"
|
|
exit 1
|
|
}
|
|
|
|
# /tmp is a tmpfs on the router, and tens of megabytes in RAM are wasted there
|
|
cleanup() {
|
|
rm -rf "$WORK" 2>/dev/null
|
|
# Files staged next to the target must not outlive an interrupted run:
|
|
# they are only renamed into place by a run that got that far.
|
|
rm -f "$DIR"/.*.new "$DIR"/.*.restore "$BACKUP_DIR"/.*.bak.new 2>/dev/null
|
|
}
|
|
trap cleanup EXIT
|
|
# A trap that only runs the cleanup does not end the script: dash and ash go
|
|
# on with the next command after the handler returns. Exit as well, or a
|
|
# cancelled update keeps downloading in the background.
|
|
trap 'exit 1' INT TERM
|
|
|
|
file_hash() {
|
|
sha256sum "$1" 2>/dev/null | awk '{print $1}'
|
|
}
|
|
|
|
# Resolve the release branch to the commit it points at and set BASE from it.
|
|
# A failed lookup (offline, GitHub rate limit) falls back to the branch: the
|
|
# run then keeps the cache race it had before, which is safer than not
|
|
# updating at all.
|
|
resolve_source() {
|
|
local commit
|
|
|
|
commit="$(curl -fsS --max-time 60 \
|
|
"https://api.github.com/repos/$REPO/branches/$BRANCH" 2>/dev/null |
|
|
sed -n 's/.*"sha"[[:space:]]*:[[:space:]]*"\([0-9a-f]\{40\}\)".*/\1/p' |
|
|
head -n 1)"
|
|
case "$commit" in
|
|
*[!0-9a-f]*|'') commit="" ;;
|
|
esac
|
|
if [ -n "$commit" ]; then
|
|
BASE="https://$HOST/gh/$REPO@$commit"
|
|
log "source: $REPO@$BRANCH is $commit"
|
|
else
|
|
BASE="https://$HOST/gh/$REPO@$BRANCH"
|
|
log "warning: cannot resolve $BRANCH to a commit, fetching the branch (the cache race stays)"
|
|
fi
|
|
}
|
|
|
|
# Fetch a file: resume an existing partial one, otherwise ask for the range
|
|
# explicitly.
|
|
fetch_file() {
|
|
local f="$1" url
|
|
url="$BASE/$f"
|
|
if [ -s "$WORK/$f" ]; then
|
|
curl -fsS -C - --connect-timeout 20 --max-time "$FETCH_TIMEOUT" -o "$WORK/$f" "$url" 2>/dev/null
|
|
else
|
|
curl -fsS -r 0- --connect-timeout 20 --max-time "$FETCH_TIMEOUT" -o "$WORK/$f" "$url" 2>/dev/null
|
|
fi
|
|
}
|
|
|
|
# Fetch a file and verify it. Returns 0 when the file is usable.
|
|
#
|
|
# The transfer and the check are separate outcomes. "curl" returning 0 means
|
|
# the file is complete; a non-zero exit means it is not, and whatever arrived
|
|
# is kept so the next round can resume it. Only a complete file that fails
|
|
# verification is deleted, because resuming it cannot help.
|
|
fetch_verified() {
|
|
local f="$1" base expected got round complete size magic
|
|
|
|
base="$BASE"
|
|
|
|
# The sha256 the upstream publishes is the only trustworthy reference
|
|
curl -fsS --max-time 60 -o "$WORK/$f.sha256sum" "$base/$f.sha256sum" 2>/dev/null
|
|
expected="$(awk '{print $1}' "$WORK/$f.sha256sum" 2>/dev/null | head -n 1)"
|
|
[ -n "$expected" ] || log "$f: no upstream sha256, size and header are the only checks left"
|
|
|
|
round=1
|
|
while [ "$round" -le "$MAX_ROUNDS" ]; do
|
|
complete=0
|
|
if fetch_file "$f"; then
|
|
complete=1
|
|
else
|
|
log "$f: round $round did not finish, keeping the partial file to resume"
|
|
fi
|
|
|
|
got="$(file_hash "$WORK/$f")"
|
|
if [ -n "$expected" ] && [ -n "$got" ] && [ "$got" = "$expected" ]; then
|
|
log "$f: sha256 matches upstream ($got)"
|
|
return 0
|
|
fi
|
|
|
|
if [ -n "$expected" ]; then
|
|
# An unfinished transfer is resumed, not discarded.
|
|
[ "$complete" = 1 ] || { round=$((round + 1)); continue; }
|
|
log "$f: round $round sha256 mismatch (local ${got:-none}, upstream $expected), fetching again"
|
|
# A complete but wrong file is not usable and cannot be resumed.
|
|
rm -f "$WORK/$f"
|
|
else
|
|
if [ "$complete" = 1 ]; then
|
|
size="$(wc -c < "$WORK/$f" 2>/dev/null)"
|
|
magic="$(hexdump -n 1 -e '1/1 "%02x"' "$WORK/$f" 2>/dev/null)"
|
|
if [ "$size" -ge "$MIN_BYTES" ] && [ "$size" -le "$MAX_BYTES" ] && [ "$magic" = "0a" ]; then
|
|
log "$f: no upstream sha256, accepted on size ($size) and header"
|
|
return 0
|
|
fi
|
|
log "$f: round $round size or header out of line ($size bytes, first byte '$magic'), fetching again"
|
|
rm -f "$WORK/$f"
|
|
fi
|
|
fi
|
|
round=$((round + 1))
|
|
done
|
|
|
|
return 1
|
|
}
|
|
|
|
log "--- start ---"
|
|
|
|
mkdir -p "$WORK" || die download_failed "cannot create the work directory $WORK"
|
|
mkdir -p "$BACKUP_DIR" || die download_failed "cannot create the backup directory $BACKUP_DIR"
|
|
|
|
# One run at a time.
|
|
#
|
|
# The work directory is a fixed path, so two runs share it: one overwrites the
|
|
# file the other is still downloading and the sha256 of both stops matching.
|
|
# The failure looks like a broken upstream rather than like a collision, and
|
|
# both runs give up. A lock on the fixed work path is enough: flock is in base,
|
|
# and the file descriptor stays open for the whole run.
|
|
mkdir -p "$(dirname "$LOCKFILE")" 2>/dev/null
|
|
if ! { exec 9>"$LOCKFILE"; } 2>/dev/null; then
|
|
die download_failed "cannot create the lock file $LOCKFILE"
|
|
fi
|
|
if ! flock -n 9; then
|
|
# The run that holds the lock owns the state file; this one must not
|
|
# overwrite it.
|
|
log "skipped: another update is already running"
|
|
exit 0
|
|
fi
|
|
|
|
set_state running ""
|
|
|
|
# The source is pinned once, before the first download: the two files and their
|
|
# sha256 have to come from the same snapshot.
|
|
resolve_source
|
|
|
|
# Both files are fetched and verified before either is installed: a failure
|
|
# halfway through leaves the running daemon on its old files.
|
|
for f in $FILES; do
|
|
fetch_verified "$f" || die download_failed "$f: download or verification failed, no file is replaced"
|
|
done
|
|
|
|
# A file that is already the newest one is left alone, so nothing has to be
|
|
# restarted for it. The other files are installed in two phases: first every
|
|
# new file is copied next to its target, then each copy is renamed into place.
|
|
# The work directory is a tmpfs and the target is on flash, so a direct "mv"
|
|
# would copy and delete, and a power loss in the middle would leave a
|
|
# truncated file where dae reads it. A copy plus a rename on the target
|
|
# filesystem cannot leave a partial file behind.
|
|
CHANGED=""
|
|
for f in $FILES; do
|
|
new="$(file_hash "$WORK/$f")"
|
|
old="$(file_hash "$DIR/$f")"
|
|
if [ -n "$old" ] && [ "$new" = "$old" ]; then
|
|
log "$f: already the newest ($new), left alone"
|
|
continue
|
|
fi
|
|
if ! cp -f "$WORK/$f" "$DIR/.$f.new"; then
|
|
rm -f "$DIR/.$f.new"
|
|
for g in $CHANGED; do rm -f "$DIR/.$g.new"; done
|
|
die download_failed "cannot stage $f"
|
|
fi
|
|
chmod 644 "$DIR/.$f.new"
|
|
CHANGED="$CHANGED $f"
|
|
done
|
|
|
|
if [ -z "$CHANGED" ]; then
|
|
set_state ok up_to_date "$(date +%s)"
|
|
log "OK: both files are already the newest, dae is not restarted"
|
|
exit 0
|
|
fi
|
|
|
|
# The backup is what the rollback restores, and it is written the same way: a
|
|
# truncated backup would be worse than no backup. Only the files this run
|
|
# replaces get a fresh .bak.
|
|
for f in $CHANGED; do
|
|
[ -f "$DIR/$f" ] || continue
|
|
if ! cp -f "$DIR/$f" "$BACKUP_DIR/.$f.bak.new"; then
|
|
rm -f "$BACKUP_DIR/.$f.bak.new"
|
|
for g in $CHANGED; do rm -f "$DIR/.$g.new"; done
|
|
die download_failed "cannot back up $f"
|
|
fi
|
|
mv -f "$BACKUP_DIR/.$f.bak.new" "$BACKUP_DIR/$f.bak" || die download_failed "cannot save the backup of $f"
|
|
done
|
|
|
|
for f in $CHANGED; do
|
|
old="$(file_hash "$DIR/$f")"
|
|
new="$(file_hash "$DIR/.$f.new")"
|
|
if ! mv -f "$DIR/.$f.new" "$DIR/$f"; then
|
|
# This rename failed; earlier iterations may already be in place, so
|
|
# put the whole set back instead of leaving a mixed pair.
|
|
log "cannot replace $f, restoring the files already replaced"
|
|
for g in $CHANGED; do
|
|
rm -f "$DIR/.$g.new"
|
|
[ -f "$BACKUP_DIR/$g.bak" ] || continue
|
|
cp -f "$BACKUP_DIR/$g.bak" "$DIR/.$g.restore" && mv -f "$DIR/.$g.restore" "$DIR/$g"
|
|
done
|
|
die download_failed "cannot replace $f"
|
|
fi
|
|
log "$f: replaced ($old -> $new)"
|
|
done
|
|
|
|
# dae reads the geo data when it starts, so a reload does not pick up the new
|
|
# files: it has to be restarted.
|
|
restart_dae() {
|
|
/etc/init.d/dae restart >/dev/null 2>&1
|
|
}
|
|
|
|
# Whether the daemon runs, asked of the init script rather than of ubus.
|
|
#
|
|
# The ubus object "dae" belongs to luci-app-dae, while this script belongs to
|
|
# the dae package. The two are installed separately: with the panel package
|
|
# absent or its rpcd backend not loaded, "ubus call dae status" fails and a
|
|
# daemon that is running perfectly well would be reported as down. That would
|
|
# roll the freshly installed geo data back on every run. The init script is
|
|
# always there, so its answer is the one to trust.
|
|
dae_running() {
|
|
/etc/init.d/dae status 2>/dev/null | grep -q '^running$'
|
|
}
|
|
|
|
# The pid of the running daemon, empty when there is none. procd stops and
|
|
# starts asynchronously, so the status word is still "running" for the old
|
|
# instance right after a restart: only a different pid proves that the new
|
|
# one is up.
|
|
dae_pid() {
|
|
pidof dae 2>/dev/null | awk '{print $1}'
|
|
}
|
|
|
|
# The service is not running: stopped by hand, disabled, or it failed to
|
|
# start. The new files are already installed and are picked up at the next
|
|
# start, so there is nothing to restart and nothing to roll back. Restarting
|
|
# here would bring up a daemon the operator stopped on purpose.
|
|
if ! dae_running; then
|
|
set_state ok updated_stopped "$(date +%s)"
|
|
log "dae is not running; the new files take effect at the next start"
|
|
exit 0
|
|
fi
|
|
|
|
before="$(dae_pid)"
|
|
restart_dae
|
|
i=0
|
|
OK=0
|
|
while [ "$i" -lt "$START_TIMEOUT" ]; do
|
|
sleep 2
|
|
pid="$(dae_pid)"
|
|
# A new pid alone is not enough: procd respawns a daemon that dies at
|
|
# once, and that instance is not a success either. It also has to report
|
|
# running for the moment it is checked.
|
|
if [ -n "$pid" ] && [ "$pid" != "$before" ] && dae_running; then
|
|
# A pid that is there is not necessarily one that stays: procd
|
|
# respawns a daemon that dies at once. The same pid after a short
|
|
# settle is the one that counts.
|
|
sleep 2
|
|
if [ "$(dae_pid)" = "$pid" ] && dae_running; then
|
|
OK=1
|
|
break
|
|
fi
|
|
fi
|
|
i=$((i + 2))
|
|
done
|
|
|
|
if [ "$OK" = "1" ]; then
|
|
set_state ok updated "$(date +%s)"
|
|
log "OK: geoip/geosite updated, dae restarted and running"
|
|
exit 0
|
|
fi
|
|
|
|
log "dae did not come up, putting back the files this run replaced"
|
|
for f in $CHANGED; do
|
|
[ -f "$BACKUP_DIR/$f.bak" ] || continue
|
|
cp -f "$BACKUP_DIR/$f.bak" "$DIR/.$f.restore" && mv -f "$DIR/.$f.restore" "$DIR/$f"
|
|
done
|
|
before="$(dae_pid)"
|
|
restart_dae
|
|
sleep 5
|
|
pid="$(dae_pid)"
|
|
if [ -n "$pid" ] && [ "$pid" != "$before" ] && dae_running; then
|
|
set_state failed restart_failed_rollback "$(date +%s)"
|
|
log "rolled back to the previous geo data, dae is running again"
|
|
else
|
|
set_state failed rollback_failed "$(date +%s)"
|
|
log "dae is still not running after the rollback, it needs a human"
|
|
fi
|
|
log "FAILED"
|
|
exit 1
|