cwx/op-packages
1
0
Fork 0
mirror of https://github.com/kiddin9/op-packages.git synced 2026-07-27 18:41:15 +08:00
Code Issues Actions Packages Projects Releases Wiki Activity
08963f8537
op-packages/luci-app-packagesync/root/usr/libexec/packagesync/sync
github-actions[bot] 32600acc60 🌴 Sync 2026-03-05 23:51:42
2026-03-05 23:51:42 +08:00

404 lines
16 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
# depends: curl getopt jsonfilter rsync ncat-full
#
# Author: muink
# Github: https://github.com/muink/luci-app-packagesync
#
# Info
PKGNAME='packagesync'
RELEASE='release'
VERSION="20240804"
#
HOMEPATH=''
JSON=''
RESULTPATH='/var/packagesync/results'
PIDFILE='/var/run/packagesync.pid'
# url path
RSYNC_URL="rsync://rsync.openwrt.org/downloads"
URLHREF_RELEASES='releases'
#URLHREF_SNAPSHOTS='snapshots'
#URLHREF_PACKAGES_PREFIX='packages-' # packages-22.03
URLHREF_VERS_PACKAGES='packages'
URLHREF_VERS_TARGETS='targets'
# rsync default options
ARCHIVE='-rlpt' # or '-a' (root)
INFO='hi'
COMPAC='--compress'
PRUNE='--delete'
#PROGR='--progress'
BWLIMIT='8000'
# Get parameters
GETOPT=$(getopt -n $(basename $0) -o d:n:s:t:k:m:e:o:l:x:V -l homedir:,name:,version:,target:,arch:,model:,extra:,orders:,bwlimit:,proxy:,rsyncurl:,Version,help -- "$@")
[ $? -ne 0 ] && >&2 echo -e "\tUse the --help option get help" && exit 1
eval set -- "$GETOPT"
OPTIONS=$(sed -E "s|'[^']*'||g; s| --( .*)?$||" <<< "$GETOPT")
# Duplicate options
for ru in --help\|--help -V\|--Version -x\|--proxy -l\|--bwlimit -d\|--homedir -n\|--name -s\|--version -t\|--target -k\|--arch -o\|--orders; do
eval "grep -qE \" ${ru%|*}[ .+]* ($ru)| ${ru#*|}[ .+]* ($ru)\" <<< \"\$OPTIONS\" && >&2 echo \"\$(basename \$0): Option '\$ru' option is repeated\" && exit 1"
done
# Independent options
for ru in --help\|--help -V\|--Version; do
eval "grep -qE \"^ ($ru) .+|.+ ($ru) .+|.+ ($ru) *\$\" <<< \"\$OPTIONS\" && >&2 echo \"\$(basename \$0): Option '\$(sed -E \"s,^.*($ru).*\$,\\1,\" <<< \"\$OPTIONS\")' cannot be used with other options\" && exit 1"
done
# Conflicting options
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-n|--name)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-n|--name'" && exit 1
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-s|--version)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-s|--version'" && exit 1
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-t|--target)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-t|--target'" && exit 1
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-k|--arch)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-k|--arch'" && exit 1
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-m|--model)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-m|--model'" && exit 1
echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-e|--extra)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-e|--extra'" && exit 1
# Required options
[ -n "$(echo "$OPTIONS" | grep -E " (-d|--homedir)\b" | grep -E " (-s|--version)\b" | grep -E " (-t|--target)\b" | grep -E " (-k|--arch)\b")" \
-o -n "$(echo "$OPTIONS" | grep -E " (-o|--orders)\b")" \
-o -n "$(echo "$OPTIONS" | grep -E " (--help|-V|--Version)\b")" ] \
|| { >&2 echo -e "$(basename $0): Missing required options\n\tUse the --help option get help"; exit 1; }
# Sub function
_help() {
printf "\n\
Usage: sync [OPTION]... \n\
\n\
e.g. sync -d /www/sync -n x64_21_02_5 -s 21.02.5 -t 'x86/64' -k x86_64\n\
e.g. sync -l 500K -d /www/sync -o \"[ {'homedir': '/www/sync', 'version': '21.02.5', 'target': 'x86/64', 'arch': 'x86_64'}, \ \n\
{'homedir': '/www/sync', 'version': '22.03.2', 'target': 'x86/64', 'arch': 'x86_64'}, \ \n\
{'homedir': '/www/sync', 'version': '21.02.5', 'target': 'ath79/nand', 'arch': 'mips_24kc'}, \ \n\
{'homedir': '/www/sync', 'version': '22.03.2', 'target': 'ath79/nand', 'arch': 'mips_24kc'} ]\" \n\
\n\
Options:\n\
-d, --homedir <homedir> -- Sync folder. Is not essential, if it has been specified in orders\n\
-n, --name <nick name> -- Identifying name. Only used when UCI/procd calls\n\
-s, --version <version> -- Required option. OpenWRT version\n\
-t, --target <target/subtarget> -- Required option. Device target and subtarget\n\
-k, --arch <pkgarch> -- Required option. Device architecture\n\
-m, --model <model> -- Optional option. Product model keyword\n\
-e, --extra <true|false> -- Optional option. Sync target supplementary. Default is false\n\
-o, --orders <orders> -- Required option. Sync multi-versions at once\n\
-l, --bwlimit <n>[(K|M|G)] -- Bandwidth limit\n\
-x, --proxy <protocol://ip:port> -- Connect using proxy\n\
--rsyncurl <url> -- Optional option. Rsync url\n\
-V, --Version -- Returns version\n\
--help -- Returns help info\n\
\n\
OrdersFormat:\n\
[ {'homedir': '??', 'name': '??', 'version': '??', 'target': '??', 'arch': '??'}, \ \n\
{'homedir': '??', 'name': '??', 'version': '??', 'target': '??', 'arch': '??'} ] \n\
'homedir' and 'name' is optional.\n\
\n"
}
_version() {
echo "$(basename $0) version: v$VERSION"
}
#verify_path <absolute path>
verify_path() {
[ -n "$1" ] || return 1
[ -d "$1" -a -n "$(grep '^/' <<< "$1")" ] || return 1
return 0
}
#verify_order <openwrt version> <target/subtarget> <package arch>
verify_order() {
local version="$1" target="$2" arch="$3"
local ERRORCODE ERROR=0
rsync --quiet --list-only "$RSYNC_URL/$URLHREF_RELEASES/${version:-null}/$URLHREF_VERS_TARGETS/${target:-null}" 2>/dev/null || { ERRORCODE=$?;
>&2 echo -e "$(basename $0): Get '$RSYNC_URL/$URLHREF_RELEASES/${version:-<null>}/$URLHREF_VERS_TARGETS/${target:-<null>}' failed. Code '$ERRORCODE'"; ((ERROR++)); }
rsync --quiet --list-only "$RSYNC_URL/$URLHREF_RELEASES/${version:-null}/$URLHREF_VERS_PACKAGES/${arch:-null}" 2>/dev/null || { ERRORCODE=$?;
>&2 echo -e "$(basename $0): Get '$RSYNC_URL/$URLHREF_RELEASES/${version:-<null>}/$URLHREF_VERS_PACKAGES/${arch:-<null>}' failed. Code '$ERRORCODE'"; ((ERROR++)); }
[ "$ERROR" -gt 0 ] && return 1
return 0
}
#parse_json <JSON>
parse_json() {
local array="$1"
[ -z "$(jsonfilter -q -s "$array" -e '$')" ] && >&2 echo -e "$(basename $0): The format of <orders> is incorrect" && return 1
local count=$(jsonfilter -q -s "$array" -e '$[*]'|sed -n '$=')
[ -z "$count" ] && >&2 echo -e "$(basename $0): The <orders> has no content" && return 1
for i in $(seq 0 $[ $count -1 ]); do
local order="$(eval "jsonfilter -q -s \"\$array\" -e '\$[$i]'")"
local ucivv="homedir version target arch model extra name"
for _var in $ucivv; do
eval "local $_var=\"\$(jsonfilter -q -s \"\$order\" -e '\$.$_var')\""
done
mkdir -p "$RESULTPATH" 2>/dev/null
if [ -n "$name" ]; then
cat /dev/null > "$RESULTPATH/${name}.log"
{ mirror "$homedir" "$version" "$target" "$arch" "$model" "$extra" 2>&1 1>&3 | tee "$RESULTPATH/${name}.log"; } 3>&1 1>&2 # in bash
#mirror "$homedir" "$version" "$target" "$arch" 2> >(tee "$RESULTPATH/${name}.log" >&2) # in POSIX shell
else
mirror "$homedir" "$version" "$target" "$arch" "$model" "$extra"
fi
local ERROR="$?"
if [ -n "$name" ]; then
local num=$(uci -q show $PKGNAME|sed -En "s|.+@$RELEASE\[(\d+)\]\.name='$name'|\1|p")
if [ -n "$num" ]; then
if [ "$ERROR" -eq 0 ]; then
uci set $PKGNAME.@$RELEASE[$num].return="Success $(date +"%FT%TZ%z")"
else
uci set $PKGNAME.@$RELEASE[$num].return="Error $(date +"%FT%TZ%z")"
#uci set $PKGNAME.@$RELEASE[$num].return="$(cat "$RESULTPATH/$name"|awk BEGIN{RS=EOF}'{gsub(/\n/,"\\\\n");print}'|sed 's|\\\\n|\\n|g')"
## |sed ':a;N;s/\n/\\\\n/;ta;'|sed 's|\\\\n|\\n|g'
fi
uci commit $PKGNAME
fi
fi
done
return $ERROR
}
#mirror <local mirror root path> <openwrt version> <target/subtarget> <package arch> <model> <extra>
mirror() {
local homedir version target arch model extra ERROR=0
homedir="${1:-$HOMEPATH}" && shift
version="$1" && shift
target="$1" && shift
arch="$1" && shift
model="$1" && shift
extra="$1" && shift
verify_path "$homedir" || { >&2 echo -e "$(basename $0): No valid <homedir> specified\n\tUse the --help option get help"; ((ERROR++)); }
verify_order "$version" "$target" "$arch" || { >&2 echo -e "$(basename $0): Order '$version' '$target' '$arch' seems invalid.\n\tPlease check for network or typos"; ((ERROR++)); }
[ "$ERROR" -gt 0 ] && return 1
#start sync
echo -e "$(basename $0): '$version' '$arch' package syncing..."
sync_package "${homedir}" "${version}" "${arch}" || ((ERROR++))
echo -e "$(basename $0): '$version' '$target'${kversion:+ 'k${kversion}'} target syncing..."
sync_target "${homedir}" "${version}" "${target}" "${kversion}" || ((ERROR++))
[ "$extra" == "true" ] && {
echo -e "$(basename $0): '$version' '$target' supplementary syncing..."
sync_extra "${homedir}" "${version}" "${target}" || ((ERROR++))
}
[ -n "$model" ] && {
echo -e "$(basename $0): '$version' '$target' '$model' model syncing..."
sync_model "${homedir}" "${version}" "${target}" "${model}" || ((ERROR++))
}
return $ERROR
}
#sync_package <local mirror root path> <openwrt version> <package arch>
sync_package() {
local home="${1%/}" && shift
local ver="$1" && shift
local arch="$1" && shift
local ERROR=0
local VER="$URLHREF_RELEASES/$ver"
mkdir -p "$home/${VER}" 2>/dev/null
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include "$URLHREF_VERS_PACKAGES" --exclude='*' "$RSYNC_URL/${VER}/" "$home/${VER}/" || ((ERROR++))
sleep 5
mkdir -p "$home/${VER}/$(ls -l "$home/${VER}/$URLHREF_VERS_PACKAGES" | sed "s|.*->[ ]*\(.*\)[ ]*$|\1|")" 2>/dev/null
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_PACKAGES/$arch" "$home/${VER}/$URLHREF_VERS_PACKAGES/" || ((ERROR++))
return $ERROR
}
#sync_target <local mirror root path> <openwrt version> <target/subtarget> [Kernel version]
sync_target() {
local home="${1%/}" && shift
local ver="$1" && shift
local target="$1" && shift
[ "$#" -ge 1 ] && local kver="$1" && shift
local ERROR=0
local VER="$URLHREF_RELEASES/$ver"
mkdir -p "$home/${VER}" 2>/dev/null
mkdir -p "$home/${VER}/$URLHREF_VERS_TARGETS/$target" 2>/dev/null
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include '*/' --exclude='*' "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++))
#kmods
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/kmods/$kver" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/kmods/" || ((ERROR++))
#packages
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/packages" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++))
return $ERROR
}
#sync_extra <local mirror root path> <openwrt version> <target/subtarget>
sync_extra() {
local home="${1%/}" && shift
local ver="$1" && shift
local target="$1" && shift
local ERROR=0
local VER="$URLHREF_RELEASES/$ver"
mkdir -p "$home/${VER}" 2>/dev/null
mkdir -p "$home/${VER}/$URLHREF_VERS_TARGETS/$target" 2>/dev/null
#supplementary
local table="$(rsync --list-only "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" | awk '{print $NF}' \
| grep -Ev '^\.{0,2}/?$' \
| grep -Ev '^(kmods|packages)/?$' \
| grep -Ev "^.*-${ver}-${target/\//-}-.*$" \
)"
if [ -n "$table" ]; then
local expr="{$(echo "$table" | sed "s|^|'|;s|$|'|" | tr '\n' ',' | sed 's|,$||')}"
eval "rsync \$ARCHIVE\$INFO \$COMPAC \$PRUNE \$PROGR --bwlimit=\$BWLIMIT --include=$expr --exclude='*' \"\$RSYNC_URL/\${VER}/\$URLHREF_VERS_TARGETS/\$target/\" \"\$home/\${VER}/\$URLHREF_VERS_TARGETS/\$target/\"" || ((ERROR++))
#for v in $(echo "$table" | sed "s|^|'|;s|$|'|"); do
# rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/${v:1:-1}" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++))
#done
else
>&2 echo -e "$(basename $0): Sync '${VER}/$URLHREF_VERS_TARGETS/$target' error. Get Supplementary Files failed.\n\tPlease check for network"
return 1
fi
return $ERROR
}
#sync_model <local mirror root path> <openwrt version> <target/subtarget> <model>
sync_model() {
local home="${1%/}" && shift
local ver="$1" && shift
local target="$1" && shift
local model="$1" && shift
local ERROR=0
#Predefined
[ -z "$model" ] && {
case "$target" in
x86/64)
model="x86-64"
;;
*)
model="${target/\//-}"
#return 1
return 0
;;
esac
}
local VER="$URLHREF_RELEASES/$ver"
mkdir -p "$home/${VER}" 2>/dev/null
rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include="*${model}-*" --exclude='*' "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++))
return $ERROR
}
# Main
# Get options
while [ -n "$1" ]; do
case "$1" in
--help)
_help
exit
;;
-V|--Version)
_version
exit
;;
-x|--proxy)
tmp=$(echo "${2,,}"|sed -En "\
/^(http|https|socks5|socks5h):\/\/\
(\
((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\
|\[(\
([[:xdigit:]]{1,4}:){7}[[:xdigit:]]{1,4}|\
([[:xdigit:]]{1,4}:){1,7}:|\
([[:xdigit:]]{1,4}:){1,6}:[[:xdigit:]]{1,4}|\
([[:xdigit:]]{1,4}:){1,5}(:[[:xdigit:]]{1,4}){1,2}|\
([[:xdigit:]]{1,4}:){1,4}(:[[:xdigit:]]{1,4}){1,3}|\
([[:xdigit:]]{1,4}:){1,3}(:[[:xdigit:]]{1,4}){1,4}|\
([[:xdigit:]]{1,4}:){1,2}(:[[:xdigit:]]{1,4}){1,5}|\
[[:xdigit:]]{1,4}:(:[[:xdigit:]]{1,4}){1,6}|\
:((:[[:xdigit:]]{1,4}){1,7}|:)|\
fe80:(:[[:xdigit:]]{0,4}){0,4}%\w+|\
::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)|\
([[:xdigit:]]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\
)\]\
)\
:(6553[0-5]|(655[0-2]|65[0-4]\d|6[0-4]\d{2}|[1-5]{0,1}\d{1,3}){0,1}\d)$/p")
if [ -n "$tmp" ]; then
export ALL_PROXY="$2"
[ -n "$(echo "${2,,}"|sed -En '/^https?:/p')" ] && export RSYNC_PROXY="$(echo "$2"|sed -En 's|^.+://||p')"
#[ -n "$(echo "${2,,}"|sed -En '/^socks5:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns local --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873"
#[ -n "$(echo "${2,,}"|sed -En '/^socks5h:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns remote --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873"
[ -n "$(echo "${2,,}"|sed -En '/^socks5h?:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns both --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873"
else
>&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help"
exit 1
fi
shift
;;
-l|--bwlimit)
[ -n "$(echo "${2^^}"|sed -En '/^(\d|[1-9]\d*(K|M|G)?)$/p')" ] && BWLIMIT="${2^^}" || { >&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help"; exit 1; }
#[ -n "$(echo "$2"|sed -En '/^(\d|[1-9]\d*(K|M|G)?)$/lp')" ]
shift
;;
--rsyncurl)
RSYNC_URL="$2"
shift
;;
-d|--homedir)
verify_path "$2" && HOMEPATH="$2" || { >&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help"; exit 1; }
shift
;;
-n|--name)
name="$2"
shift
;;
-s|--version)
version="$2"
shift
;;
-t|--target)
target="$2"
shift
;;
-k|--arch)
arch="$2"
shift
;;
-m|--model)
model="$2"
shift
;;
-e|--extra)
extra="$2"
shift
;;
-o|--orders)
JSON="$2"
shift
;;
--)
shift
break
;;
*)
>&2 echo -e "$(basename $0): '$1' is not a valid option\n\tUse the --help option get help"
exit 1
;;
esac
shift
done
# Get parameters
# ...
# Execute
parse_json "${JSON:=[{'name':'$name','version':'$version','target':'$target','arch':'$arch','model':'${model}','extra':${extra:-false}\}]}" #|| { >&2 echo -e "$(basename $0): Required options requires valid argument\n\tUse the --help option get help"; exit 1; }
exitcode=$?
rm -f "$PIDFILE" 2>/dev/null
exit $exitcode
Reference in New Issue View Git Blame Copy Permalink