#!/bin/bash # depends: curl getopt jsonfilter rsync ncat-full # # Author: muink # Github: https://github.com/muink/luci-app-packagesync # # Info PKGNAME='packagesync' RELEASE='release' VERSION="20240804" # HOMEPATH='' JSON='' RESULTPATH='/var/packagesync/results' PIDFILE='/var/run/packagesync.pid' # url path RSYNC_URL="rsync://rsync.openwrt.org/downloads" URLHREF_RELEASES='releases' #URLHREF_SNAPSHOTS='snapshots' #URLHREF_PACKAGES_PREFIX='packages-' # packages-22.03 URLHREF_VERS_PACKAGES='packages' URLHREF_VERS_TARGETS='targets' # rsync default options ARCHIVE='-rlpt' # or '-a' (root) INFO='hi' COMPAC='--compress' PRUNE='--delete' #PROGR='--progress' BWLIMIT='8000' # Get parameters GETOPT=$(getopt -n $(basename $0) -o d:n:s:t:k:m:e:o:l:x:V -l homedir:,name:,version:,target:,arch:,model:,extra:,orders:,bwlimit:,proxy:,rsyncurl:,Version,help -- "$@") [ $? -ne 0 ] && >&2 echo -e "\tUse the --help option get help" && exit 1 eval set -- "$GETOPT" OPTIONS=$(sed -E "s|'[^']*'||g; s| --( .*)?$||" <<< "$GETOPT") # Duplicate options for ru in --help\|--help -V\|--Version -x\|--proxy -l\|--bwlimit -d\|--homedir -n\|--name -s\|--version -t\|--target -k\|--arch -o\|--orders; do eval "grep -qE \" ${ru%|*}[ .+]* ($ru)| ${ru#*|}[ .+]* ($ru)\" <<< \"\$OPTIONS\" && >&2 echo \"\$(basename \$0): Option '\$ru' option is repeated\" && exit 1" done # Independent options for ru in --help\|--help -V\|--Version; do eval "grep -qE \"^ ($ru) .+|.+ ($ru) .+|.+ ($ru) *\$\" <<< \"\$OPTIONS\" && >&2 echo \"\$(basename \$0): Option '\$(sed -E \"s,^.*($ru).*\$,\\1,\" <<< \"\$OPTIONS\")' cannot be used with other options\" && exit 1" done # Conflicting options echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-n|--name)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-n|--name'" && exit 1 echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-s|--version)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-s|--version'" && exit 1 echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-t|--target)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-t|--target'" && exit 1 echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-k|--arch)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-k|--arch'" && exit 1 echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-m|--model)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-m|--model'" && exit 1 echo "$OPTIONS" | grep -E " (-o|--orders)\b" | grep -E " (-e|--extra)\b" >/dev/null && >&2 echo "$(basename $0): Option '-o|--orders' cannot be used with option '-e|--extra'" && exit 1 # Required options [ -n "$(echo "$OPTIONS" | grep -E " (-d|--homedir)\b" | grep -E " (-s|--version)\b" | grep -E " (-t|--target)\b" | grep -E " (-k|--arch)\b")" \ -o -n "$(echo "$OPTIONS" | grep -E " (-o|--orders)\b")" \ -o -n "$(echo "$OPTIONS" | grep -E " (--help|-V|--Version)\b")" ] \ || { >&2 echo -e "$(basename $0): Missing required options\n\tUse the --help option get help"; exit 1; } # Sub function _help() { printf "\n\ Usage: sync [OPTION]... \n\ \n\ e.g. sync -d /www/sync -n x64_21_02_5 -s 21.02.5 -t 'x86/64' -k x86_64\n\ e.g. sync -l 500K -d /www/sync -o \"[ {'homedir': '/www/sync', 'version': '21.02.5', 'target': 'x86/64', 'arch': 'x86_64'}, \ \n\ {'homedir': '/www/sync', 'version': '22.03.2', 'target': 'x86/64', 'arch': 'x86_64'}, \ \n\ {'homedir': '/www/sync', 'version': '21.02.5', 'target': 'ath79/nand', 'arch': 'mips_24kc'}, \ \n\ {'homedir': '/www/sync', 'version': '22.03.2', 'target': 'ath79/nand', 'arch': 'mips_24kc'} ]\" \n\ \n\ Options:\n\ -d, --homedir -- Sync folder. Is not essential, if it has been specified in orders\n\ -n, --name -- Identifying name. Only used when UCI/procd calls\n\ -s, --version -- Required option. OpenWRT version\n\ -t, --target -- Required option. Device target and subtarget\n\ -k, --arch -- Required option. Device architecture\n\ -m, --model -- Optional option. Product model keyword\n\ -e, --extra -- Optional option. Sync target supplementary. Default is false\n\ -o, --orders -- Required option. Sync multi-versions at once\n\ -l, --bwlimit [(K|M|G)] -- Bandwidth limit\n\ -x, --proxy -- Connect using proxy\n\ --rsyncurl -- Optional option. Rsync url\n\ -V, --Version -- Returns version\n\ --help -- Returns help info\n\ \n\ OrdersFormat:\n\ [ {'homedir': '??', 'name': '??', 'version': '??', 'target': '??', 'arch': '??'}, \ \n\ {'homedir': '??', 'name': '??', 'version': '??', 'target': '??', 'arch': '??'} ] \n\ 'homedir' and 'name' is optional.\n\ \n" } _version() { echo "$(basename $0) version: v$VERSION" } #verify_path verify_path() { [ -n "$1" ] || return 1 [ -d "$1" -a -n "$(grep '^/' <<< "$1")" ] || return 1 return 0 } #verify_order verify_order() { local version="$1" target="$2" arch="$3" local ERRORCODE ERROR=0 rsync --quiet --list-only "$RSYNC_URL/$URLHREF_RELEASES/${version:-null}/$URLHREF_VERS_TARGETS/${target:-null}" 2>/dev/null || { ERRORCODE=$?; >&2 echo -e "$(basename $0): Get '$RSYNC_URL/$URLHREF_RELEASES/${version:-}/$URLHREF_VERS_TARGETS/${target:-}' failed. Code '$ERRORCODE'"; ((ERROR++)); } rsync --quiet --list-only "$RSYNC_URL/$URLHREF_RELEASES/${version:-null}/$URLHREF_VERS_PACKAGES/${arch:-null}" 2>/dev/null || { ERRORCODE=$?; >&2 echo -e "$(basename $0): Get '$RSYNC_URL/$URLHREF_RELEASES/${version:-}/$URLHREF_VERS_PACKAGES/${arch:-}' failed. Code '$ERRORCODE'"; ((ERROR++)); } [ "$ERROR" -gt 0 ] && return 1 return 0 } #parse_json parse_json() { local array="$1" [ -z "$(jsonfilter -q -s "$array" -e '$')" ] && >&2 echo -e "$(basename $0): The format of is incorrect" && return 1 local count=$(jsonfilter -q -s "$array" -e '$[*]'|sed -n '$=') [ -z "$count" ] && >&2 echo -e "$(basename $0): The has no content" && return 1 for i in $(seq 0 $[ $count -1 ]); do local order="$(eval "jsonfilter -q -s \"\$array\" -e '\$[$i]'")" local ucivv="homedir version target arch model extra name" for _var in $ucivv; do eval "local $_var=\"\$(jsonfilter -q -s \"\$order\" -e '\$.$_var')\"" done mkdir -p "$RESULTPATH" 2>/dev/null if [ -n "$name" ]; then cat /dev/null > "$RESULTPATH/${name}.log" { mirror "$homedir" "$version" "$target" "$arch" "$model" "$extra" 2>&1 1>&3 | tee "$RESULTPATH/${name}.log"; } 3>&1 1>&2 # in bash #mirror "$homedir" "$version" "$target" "$arch" 2> >(tee "$RESULTPATH/${name}.log" >&2) # in POSIX shell else mirror "$homedir" "$version" "$target" "$arch" "$model" "$extra" fi local ERROR="$?" if [ -n "$name" ]; then local num=$(uci -q show $PKGNAME|sed -En "s|.+@$RELEASE\[(\d+)\]\.name='$name'|\1|p") if [ -n "$num" ]; then if [ "$ERROR" -eq 0 ]; then uci set $PKGNAME.@$RELEASE[$num].return="Success $(date +"%FT%TZ%z")" else uci set $PKGNAME.@$RELEASE[$num].return="Error $(date +"%FT%TZ%z")" #uci set $PKGNAME.@$RELEASE[$num].return="$(cat "$RESULTPATH/$name"|awk BEGIN{RS=EOF}'{gsub(/\n/,"\\\\n");print}'|sed 's|\\\\n|\\n|g')" ## |sed ':a;N;s/\n/\\\\n/;ta;'|sed 's|\\\\n|\\n|g' fi uci commit $PKGNAME fi fi done return $ERROR } #mirror mirror() { local homedir version target arch model extra ERROR=0 homedir="${1:-$HOMEPATH}" && shift version="$1" && shift target="$1" && shift arch="$1" && shift model="$1" && shift extra="$1" && shift verify_path "$homedir" || { >&2 echo -e "$(basename $0): No valid specified\n\tUse the --help option get help"; ((ERROR++)); } verify_order "$version" "$target" "$arch" || { >&2 echo -e "$(basename $0): Order '$version' '$target' '$arch' seems invalid.\n\tPlease check for network or typos"; ((ERROR++)); } [ "$ERROR" -gt 0 ] && return 1 #start sync echo -e "$(basename $0): '$version' '$arch' package syncing..." sync_package "${homedir}" "${version}" "${arch}" || ((ERROR++)) echo -e "$(basename $0): '$version' '$target'${kversion:+ 'k${kversion}'} target syncing..." sync_target "${homedir}" "${version}" "${target}" "${kversion}" || ((ERROR++)) [ "$extra" == "true" ] && { echo -e "$(basename $0): '$version' '$target' supplementary syncing..." sync_extra "${homedir}" "${version}" "${target}" || ((ERROR++)) } [ -n "$model" ] && { echo -e "$(basename $0): '$version' '$target' '$model' model syncing..." sync_model "${homedir}" "${version}" "${target}" "${model}" || ((ERROR++)) } return $ERROR } #sync_package sync_package() { local home="${1%/}" && shift local ver="$1" && shift local arch="$1" && shift local ERROR=0 local VER="$URLHREF_RELEASES/$ver" mkdir -p "$home/${VER}" 2>/dev/null rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include "$URLHREF_VERS_PACKAGES" --exclude='*' "$RSYNC_URL/${VER}/" "$home/${VER}/" || ((ERROR++)) sleep 5 mkdir -p "$home/${VER}/$(ls -l "$home/${VER}/$URLHREF_VERS_PACKAGES" | sed "s|.*->[ ]*\(.*\)[ ]*$|\1|")" 2>/dev/null rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_PACKAGES/$arch" "$home/${VER}/$URLHREF_VERS_PACKAGES/" || ((ERROR++)) return $ERROR } #sync_target [Kernel version] sync_target() { local home="${1%/}" && shift local ver="$1" && shift local target="$1" && shift [ "$#" -ge 1 ] && local kver="$1" && shift local ERROR=0 local VER="$URLHREF_RELEASES/$ver" mkdir -p "$home/${VER}" 2>/dev/null mkdir -p "$home/${VER}/$URLHREF_VERS_TARGETS/$target" 2>/dev/null rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include '*/' --exclude='*' "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++)) #kmods rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/kmods/$kver" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/kmods/" || ((ERROR++)) #packages rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/packages" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++)) return $ERROR } #sync_extra sync_extra() { local home="${1%/}" && shift local ver="$1" && shift local target="$1" && shift local ERROR=0 local VER="$URLHREF_RELEASES/$ver" mkdir -p "$home/${VER}" 2>/dev/null mkdir -p "$home/${VER}/$URLHREF_VERS_TARGETS/$target" 2>/dev/null #supplementary local table="$(rsync --list-only "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" | awk '{print $NF}' \ | grep -Ev '^\.{0,2}/?$' \ | grep -Ev '^(kmods|packages)/?$' \ | grep -Ev "^.*-${ver}-${target/\//-}-.*$" \ )" if [ -n "$table" ]; then local expr="{$(echo "$table" | sed "s|^|'|;s|$|'|" | tr '\n' ',' | sed 's|,$||')}" eval "rsync \$ARCHIVE\$INFO \$COMPAC \$PRUNE \$PROGR --bwlimit=\$BWLIMIT --include=$expr --exclude='*' \"\$RSYNC_URL/\${VER}/\$URLHREF_VERS_TARGETS/\$target/\" \"\$home/\${VER}/\$URLHREF_VERS_TARGETS/\$target/\"" || ((ERROR++)) #for v in $(echo "$table" | sed "s|^|'|;s|$|'|"); do # rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/${v:1:-1}" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++)) #done else >&2 echo -e "$(basename $0): Sync '${VER}/$URLHREF_VERS_TARGETS/$target' error. Get Supplementary Files failed.\n\tPlease check for network" return 1 fi return $ERROR } #sync_model sync_model() { local home="${1%/}" && shift local ver="$1" && shift local target="$1" && shift local model="$1" && shift local ERROR=0 #Predefined [ -z "$model" ] && { case "$target" in x86/64) model="x86-64" ;; *) model="${target/\//-}" #return 1 return 0 ;; esac } local VER="$URLHREF_RELEASES/$ver" mkdir -p "$home/${VER}" 2>/dev/null rsync $ARCHIVE$INFO $COMPAC $PRUNE $PROGR --bwlimit=$BWLIMIT --include="*${model}-*" --exclude='*' "$RSYNC_URL/${VER}/$URLHREF_VERS_TARGETS/$target/" "$home/${VER}/$URLHREF_VERS_TARGETS/$target/" || ((ERROR++)) return $ERROR } # Main # Get options while [ -n "$1" ]; do case "$1" in --help) _help exit ;; -V|--Version) _version exit ;; -x|--proxy) tmp=$(echo "${2,,}"|sed -En "\ /^(http|https|socks5|socks5h):\/\/\ (\ ((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\ |\[(\ ([[:xdigit:]]{1,4}:){7}[[:xdigit:]]{1,4}|\ ([[:xdigit:]]{1,4}:){1,7}:|\ ([[:xdigit:]]{1,4}:){1,6}:[[:xdigit:]]{1,4}|\ ([[:xdigit:]]{1,4}:){1,5}(:[[:xdigit:]]{1,4}){1,2}|\ ([[:xdigit:]]{1,4}:){1,4}(:[[:xdigit:]]{1,4}){1,3}|\ ([[:xdigit:]]{1,4}:){1,3}(:[[:xdigit:]]{1,4}){1,4}|\ ([[:xdigit:]]{1,4}:){1,2}(:[[:xdigit:]]{1,4}){1,5}|\ [[:xdigit:]]{1,4}:(:[[:xdigit:]]{1,4}){1,6}|\ :((:[[:xdigit:]]{1,4}){1,7}|:)|\ fe80:(:[[:xdigit:]]{0,4}){0,4}%\w+|\ ::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)|\ ([[:xdigit:]]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\.){3}(25[0-5]|(2[0-4]|1{0,1}\d){0,1}\d)\ )\]\ )\ :(6553[0-5]|(655[0-2]|65[0-4]\d|6[0-4]\d{2}|[1-5]{0,1}\d{1,3}){0,1}\d)$/p") if [ -n "$tmp" ]; then export ALL_PROXY="$2" [ -n "$(echo "${2,,}"|sed -En '/^https?:/p')" ] && export RSYNC_PROXY="$(echo "$2"|sed -En 's|^.+://||p')" #[ -n "$(echo "${2,,}"|sed -En '/^socks5:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns local --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873" #[ -n "$(echo "${2,,}"|sed -En '/^socks5h:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns remote --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873" [ -n "$(echo "${2,,}"|sed -En '/^socks5h?:/p')" ] && export RSYNC_CONNECT_PROG="ncat --proxy-type socks5 --proxy-dns both --proxy $(echo "$2"|sed -En 's|^.+://||p') %H 873" else >&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help" exit 1 fi shift ;; -l|--bwlimit) [ -n "$(echo "${2^^}"|sed -En '/^(\d|[1-9]\d*(K|M|G)?)$/p')" ] && BWLIMIT="${2^^}" || { >&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help"; exit 1; } #[ -n "$(echo "$2"|sed -En '/^(\d|[1-9]\d*(K|M|G)?)$/lp')" ] shift ;; --rsyncurl) RSYNC_URL="$2" shift ;; -d|--homedir) verify_path "$2" && HOMEPATH="$2" || { >&2 echo -e "$(basename $0): Option '$1' requires a valid argument\n\tUse the --help option get help"; exit 1; } shift ;; -n|--name) name="$2" shift ;; -s|--version) version="$2" shift ;; -t|--target) target="$2" shift ;; -k|--arch) arch="$2" shift ;; -m|--model) model="$2" shift ;; -e|--extra) extra="$2" shift ;; -o|--orders) JSON="$2" shift ;; --) shift break ;; *) >&2 echo -e "$(basename $0): '$1' is not a valid option\n\tUse the --help option get help" exit 1 ;; esac shift done # Get parameters # ... # Execute parse_json "${JSON:=[{'name':'$name','version':'$version','target':'$target','arch':'$arch','model':'${model}','extra':${extra:-false}\}]}" #|| { >&2 echo -e "$(basename $0): Required options requires valid argument\n\tUse the --help option get help"; exit 1; } exitcode=$? rm -f "$PIDFILE" 2>/dev/null exit $exitcode