From 61d0ef38350047d72e669f8bbebd4c662e095509 Mon Sep 17 00:00:00 2001 From: WXbet Date: Sun, 19 Jan 2020 23:38:54 +0100 Subject: [PATCH] Add new commandline option to reset the config file (v0.4.0) ./s3 tcupdate -r or ./s3 tcupdate --reset Reset the config file determines the most recent versions and download links for the libraries dynamically via the Github API Rework download function to handle content disposition Remove config file from repository --- .../configs/plugin_update_toolchain.config | 130 ----------------- support/functions/_plugin_update_toolchain | 133 ++++++++++++++---- 2 files changed, 105 insertions(+), 158 deletions(-) delete mode 100644 support/configs/plugin_update_toolchain.config diff --git a/support/configs/plugin_update_toolchain.config b/support/configs/plugin_update_toolchain.config deleted file mode 100644 index 8d38547..0000000 --- a/support/configs/plugin_update_toolchain.config +++ /dev/null @@ -1,130 +0,0 @@ -#list of ordered library list -#only libraries in this list are available -LIBS=(SAMPLE_LIB SSL_111 SSL_110 SSL_102 SSL_100 SSL_098 LIB_USB LIB_PCSC LIB_ZLIB); - -#SampleLib 0.2.9sl -SAMPLE_LIB="0"; #Enable/Disable (1|0) -SAMPLE_LIB_name="SampleLib"; #Displayname -SAMPLE_LIB_version="0.2.9sl"; #Versionnumber -SAMPLE_LIB_check="samplelib.pc"; #Filename to check in pkgconfig folder for an existing library to get version information -SAMPLE_LIB_url="https://sampleurl.com/archive.tar.bz"; #Valid URL for downloading sources of library -#List of build commands below. Pay attention to the SEMICOLON at the end of EACH -#command and the correct quoting (\' or ") for or to avoid expansion of variables. -#Tokens @TOOLCHAIN@, @CC@, @RANLIB@, @PREFIX@ are replaced automatically. -SAMPLE_LIB_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); #build command1 -SAMPLE_LIB_tasks+=("export CC=@CC@;"); #build command2 -SAMPLE_LIB_tasks+=("export RANLIB=@RANLIB@;"); #build command3 -SAMPLE_LIB_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); #build command4 -SAMPLE_LIB_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); #build command5 -SAMPLE_LIB_tasks+=("make;"); #build command6 -SAMPLE_LIB_tasks+=("make install;"); #build command7 - -#OpenSSL 1.1.1d -SSL_111="1"; -SSL_111_name="OpenSSL"; -SSL_111_version="1.1.1d"; -SSL_111_check="openssl.pc"; -SSL_111_url="https://www.openssl.org/source/openssl-1.1.1d.tar.gz"; -SSL_111_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -SSL_111_tasks+=("export CC=@CC@;"); -SSL_111_tasks+=("export RANLIB=@RANLIB@;"); -SSL_111_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -SSL_111_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); -SSL_111_tasks+=("make;"); -SSL_111_tasks+=("make install;"); - -#OpenSSL 1.1.0l -SSL_110="1"; -SSL_110_name="OpenSSL"; -SSL_110_version="1.1.0l"; -SSL_110_check="openssl.pc"; -SSL_110_url="https://www.openssl.org/source/openssl-1.1.0l.tar.gz"; -SSL_110_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -SSL_110_tasks+=("export CC=@CC@;"); -SSL_110_tasks+=("export RANLIB=@RANLIB@;"); -SSL_110_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -SSL_110_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); -SSL_110_tasks+=("make;"); -SSL_110_tasks+=("make install;"); - -#OpenSSL 1.0.2u -SSL_102="1"; -SSL_102_name="OpenSSL"; -SSL_102_version="1.0.2u"; -SSL_102_check="openssl.pc"; -SSL_102_url="https://www.openssl.org/source/openssl-1.0.2u.tar.gz"; -SSL_102_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -SSL_102_tasks+=("export CC=@CC@;"); -SSL_102_tasks+=("export RANLIB=@RANLIB@;"); -SSL_102_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -SSL_102_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); -SSL_102_tasks+=("make;"); -SSL_102_tasks+=("make install SHLIB_VERSION_NUMBER=1.0.2 SHLIB_MINOR=0.2;"); - -#OpenSSL 1.0.0t -SSL_100="1"; -SSL_100_name="OpenSSL"; -SSL_100_version="1.0.0t"; -SSL_100_check="openssl.pc"; -SSL_100_url="https://www.openssl.org/source/openssl-1.0.0t.tar.gz"; -SSL_100_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -SSL_100_tasks+=("export CC=@CC@;"); -SSL_100_tasks+=("export RANLIB=@RANLIB@;"); -SSL_100_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -SSL_100_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); -SSL_100_tasks+=("make;"); -SSL_100_tasks+=("make install;"); - -#OpenSSL 0.9.8zf -SSL_098="1"; -SSL_098_name="OpenSSL"; -SSL_098_version="0.9.8zf"; -SSL_098_check="openssl.pc"; -SSL_098_url="https://www.openssl.org/source/openssl-0.9.8zf.tar.gz"; -SSL_098_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -SSL_098_tasks+=("export CC=@CC@;"); -SSL_098_tasks+=("export RANLIB=@RANLIB@;"); -SSL_098_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -SSL_098_tasks+=("./Configure --prefix=@PREFIX@/ linux-generic32 shared;"); -SSL_098_tasks+=("make;"); -SSL_098_tasks+=("make install;"); - -#libusb 1.0.23 -LIB_USB="1"; -LIB_USB_name="libusb"; -LIB_USB_version="1.0.23"; -LIB_USB_check="libusb-1.0.pc"; -LIB_USB_url="https://github.com/libusb/libusb/archive/v1.0.23.zip"; -LIB_USB_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -LIB_USB_tasks+=("export CC=@CC@;"); -LIB_USB_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -LIB_USB_tasks+=("./bootstrap.sh;"); -LIB_USB_tasks+=("./configure --prefix=@PREFIX@/ --host=@HOST@ --disable-udev;"); -LIB_USB_tasks+=("make;"); -LIB_USB_tasks+=("make install;"); - -#PCSC-Lite 1.8.26 -LIB_PCSC="1"; -LIB_PCSC_name="PCSC-Lite"; -LIB_PCSC_version="1.8.26"; -LIB_PCSC_check="libpcsclite.pc"; -LIB_PCSC_url="https://pcsclite.apdu.fr/files/pcsc-lite-1.8.26.tar.bz2"; -LIB_PCSC_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -LIB_PCSC_tasks+=("export CC=@CC@;"); -LIB_PCSC_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -LIB_PCSC_tasks+=("./configure --prefix=@PREFIX@/ --host=@HOST@ --disable-libudev --disable-libsystemd LIBUSB_LIBS=\"-L@PREFIX@/lib -lusb-1.0 -lrt\" LIBUSB_CFLAGS=\"-I@PREFIX@/include/libusb-1.0\";"); -LIB_PCSC_tasks+=("make;"); -LIB_PCSC_tasks+=("make install;"); - -#zlib 1.2.11 -LIB_ZLIB="1"; -LIB_ZLIB_name="zlib"; -LIB_ZLIB_version="1.2.11"; -LIB_ZLIB_check="zlib.pc"; -LIB_ZLIB_url="https://www.zlib.net/zlib1211.zip"; -LIB_ZLIB_tasks=("export TOOLCHAIN=@TOOLCHAIN@;"); -LIB_ZLIB_tasks+=("export CC=@CC@;"); -LIB_ZLIB_tasks+=('export PATH="$TOOLCHAIN/bin:$PATH";'); -LIB_ZLIB_tasks+=("./configure --prefix=@PREFIX@/;"); -LIB_ZLIB_tasks+=("make;"); -LIB_ZLIB_tasks+=("make install;"); diff --git a/support/functions/_plugin_update_toolchain b/support/functions/_plugin_update_toolchain index 955331b..b05d07f 100644 --- a/support/functions/_plugin_update_toolchain +++ b/support/functions/_plugin_update_toolchain @@ -1,9 +1,35 @@ #!/bin/bash -#simplebuild_plugin tcupdate 0.3.0 +#simplebuild_plugin tcupdate 0.4.0 tcupdate(){ - clear; + clear && _check_bin; + + configname="$configdir/plugin_update_toolchain.config"; + echo -e "config file='$configname'\n"; + #remove config + if [ "$1" == "-r" ] || [ "$1" == "--reset" ];then + if [ -f "$configname" ];then + if ! _check_github_api_limits 12; then + echo -e "$y_l""RESET -> Moving existing config file...""$re_"; + bcn="$configname".$(date +"%Y%m%d%H%M%S"); + mv "$configname" "$bcn" + echo -e "config file backup='$bcn'\n"; + else + _paktc_timer 30; + fi; + fi; + set -- #remove all parameters + fi; + #create config + if [ ! -f "$configname" ];then + echo -e "$g_l""CLEANUP -> Creating new config file. Please wait...""$y_l""$re_"; + _create_config; + echo -e "config file='$configname'\nYou can extend them to fit your needs.\n"; + _paktc_timer 5; + fi; + #load config + source "$configname"; if [ "$1" == "" ];then _fill_tc_array; @@ -26,7 +52,7 @@ tcupdate(){ IFS=$OIFS; [ "$out" == "" ] && bye; first=$(echo "$out" | awk '{printf $1}'); - tc_info=$(echo "$out" | awk '{printf $2}';); + tc_info=$(echo "$out" | awk '{printf $2}'); case $first in HELP) _toolchain_repair "$(_backup "$tc_info")"; @@ -58,17 +84,6 @@ tcupdate(){ exit; fi; - #create/load config - configname="$configdir/plugin_update_toolchain.config"; - if [ ! -f "$configname" ];then - echo -e "$r_l$txt_error: config file $configdir/libs not found!\n" \ - "$g_l""Creating new config file...\n" \ - "$y_l""You can extend them to fit your needs.""$re_"; - _create_config; - sleep 2; - fi; - source "$configname"; - #autodetect toolchain's sysroot, prefix and pkgconfig folder compilername="$_compiler""gcc";compilername=$(realpath $compilername); ranlibname="$_compiler""ranlib";ranlibname=$(realpath $ranlibname); @@ -103,7 +118,7 @@ tcupdate(){ libconfigure="$i"_configure[@];libconfigure=("${!libconfigure}"); libmake="$i"_make[@];libmake=("${!libmake}"); - MENU_OPTIONS+=("$i" "$libname $libversion $libversioncompare $(echo $libversioncurrent | sed -e 's/^$/ ---/g')" "$([ $libversioncurrent ] && echo "on" || echo "off")" "$([ $libname ] && echo "$libname $libversion: $liburl")");counter; + MENU_OPTIONS+=("$i" "$libname $libversion $libversioncompare $(echo $libversioncurrent | sed -e 's/^$/ ---/g')" "$([ $libversioncurrent ] && echo "on" || echo "off")" "$([ $libname ] && echo "$libname $libversion: $liburl")");counter; # 0 key 1 desc 2 version 3 checkversion 4 checkfile 5 current version 6 url 7 tasks array lib=("$libkey" "$libname" "$libversion" "$libversioncheck" "$libcheck" "$libversioncurrent" "$liburl" "(${libtasks[*]@Q})"); @@ -233,10 +248,17 @@ _build(){ }; _dl(){ - fname=$(basename "${1%..*}") - if [ ! -f "$dldir/$fname" ];then + [ -d "$dldir/#tmp" ] && rm --recursive --force "$dldir/#tmp" + mkdir --parents "$dldir/#tmp"; + cd "$dldir/#tmp"; + wget --content-disposition "$1" 2>&1 | stdbuf -o0 awk '/[.] +[0-9][0-9]?[0-9]?%/ { print substr($0,63,3) }' | "$gui" "$st_" "$bt_" "$title_" --title " -[ download ]- " --gauge " $txt_loading $fname $txt_wait" 6 74; + if [ $? -eq 0 ];then + fname="$(ls -t | head -n1)"; md5file="$fname.md5"; + md5sum "$fname" > "$md5file"; cd "$dldir"; - wget "$1" 2>&1 | stdbuf -o0 awk '/[.] +[0-9][0-9]?[0-9]?%/ { print substr($0,63,3) }' | "$gui" "$st_" "$bt_" "$title_" --title " -[ download ]- " --gauge " $txt_loading $fname $txt_wait" 6 74; + answer=$(md5sum -c "$md5file" 2>/dev/null | awk -F: '{printf $2}'); + [ ! "$answer" == "OK" ] && mv --force "$dldir/#tmp/$fname" "$dldir/$fname"; + rm --recursive --force "$dldir/#tmp"; fi; [ -f "$dldir/$fname" ] && echo "$dldir/$fname"; exit; @@ -320,6 +342,52 @@ _compare_version(){ [ $1 == $max ] && printf '>' && exit; [ $1 == $min ] && printf '<' && exit; }; +_paktc_timer(){ +if [ "$1" == "0" ] || [ "${#1}" -eq 0 ];then + read -n 1 -s -r -p "Press any key to continue"; +else + for (( i=$1; i>0; i--)); do + sp=$(printf '%*s' $[$1-$i+1] | tr ' ' '.'); + printf "\rPress any key to continue$sp" + read -s -n 1 -t 1 key + if [ $? -eq 0 ] + then + break + fi; + done +fi; +}; +_check_bin(){ + bins=( curl jq ); + failed=0; + for e in "${bins[@]}"; + do + if ! hash "$e" 2>/dev/null; then + packages="$packages $e"; + failed=1; + fi; + done; + [ "$failed" == 1 ] && echo -e "$r_l""\nCHECK -> You need to install the following tools to use this plugin properly:\n"$b_l"apt install$packages\n""$re_" && _paktc_timer && echo -e "\n" && exit; +}; +_check_github_api_limits(){ + + limit=$(curl --silent "https://api.github.com/rate_limit" | jq -r '.resources.core.limit'); + remaining=$(curl --silent "https://api.github.com/rate_limit" | jq -r '.resources.core.remaining'); + reset=$(curl --silent "https://api.github.com/rate_limit" | jq -r '.resources.core.reset'); + reset_time=$(date -d @$reset); + + if [ "$remaining" -lt "$1" ]; then + echo -e "$y_l""NOTE -> Please wait until "$g_l"$reset_time"$y_l" to reset your configuration file."\ + "The most recent versions and download links for the libraries are determined dynamically via the Github API."\ + "Exceeding the rate limit on Github API will result in failures on generationg the configuration file."\ + "Your Github API rate limit will be reset soon on $reset_time."\ + "\nFor "$b_l"unauthenticated requests"$y_l", the rate limit allows for "$b_l"up to $limit requests per hour"$y_l". Unauthenticated requests are associated with the originating IP address, and not the user making requests."\ + "\n$re_"; + return 0; + else + return 1; + fi; +}; _create_config(){ sample_comments=( "#Enable/Disable (1|0)" \ "#Displayname" \ @@ -331,17 +399,26 @@ _create_config(){ sample_settings=( "" name version check url tasks ); sample_keys=( SAMPLE_LIB SSL_111 SSL_110 SSL_102 SSL_100 SSL_098 LIB_USB LIB_PCSC LIB_ZLIB ); sample_names=( "SampleLib" "OpenSSL" "OpenSSL" "OpenSSL" "OpenSSL" "OpenSSL" "libusb" "PCSC-Lite" "zlib" ); - sample_versions=( "0.2.9sl" "1.1.1d" "1.1.0l" "1.0.2u" "1.0.0t" "0.9.8zf" "1.0.23" "1.8.26" "1.2.11" ); + sample_versions=( "0.2.9sl" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_1_1[a-z].*"))][0] | .name' | sed -e 's#OpenSSL_##g; s#_#.#g')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_1_0[a-z].*"))][0] | .name' | sed -e 's#OpenSSL_##g; s#_#.#g')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_0_2[a-z].*"))][0] | .name' | sed -e 's#OpenSSL_##g; s#_#.#g')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=2&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_0_0[a-z].*"))][0] | .name' | sed -e 's#OpenSSL_##g; s#_#.#g')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=2&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_0_9_8[a-z].*"))][0] | .name' | sed -e 's#OpenSSL_##g; s#_#.#g')" \ + "$(curl --silent "https://api.github.com/repos/libusb/libusb/releases/latest" | jq -r '.tag_name' | cut -c 2-)" \ + "$(curl --silent https://salsa.debian.org/api/v4/projects/19181/repository/tags | jq -r '.[0].name' | awk -F '[/-]' '{printf $2}')" \ + "$(curl --silent https://api.github.com/repos/madler/zlib/tags | jq -r '.[0].name' | cut -c 2-)" \ + ); sample_checks=( "samplelib.pc" "openssl.pc" "openssl.pc" "openssl.pc" "openssl.pc" "openssl.pc" "libusb-1.0.pc" "libpcsclite.pc" "zlib.pc" ); sample_urls=( "https://sampleurl.com/archive.tar.bz" \ - "https://www.openssl.org/source/openssl-1.1.1d.tar.gz" \ - "https://www.openssl.org/source/openssl-1.1.0l.tar.gz" \ - "https://www.openssl.org/source/openssl-1.0.2u.tar.gz" \ - "https://www.openssl.org/source/openssl-1.0.0t.tar.gz" \ - "https://www.openssl.org/source/openssl-0.9.8zf.tar.gz" \ - "https://github.com/libusb/libusb/archive/v1.0.23.zip" \ - "https://pcsclite.apdu.fr/files/pcsc-lite-1.8.26.tar.bz2" \ - "https://www.zlib.net/zlib1211.zip" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_1_1[a-z].*"))][0] | .tarball_url')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_1_0[a-z].*"))][0] | .tarball_url')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=1&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_0_2[a-z].*"))][0] | .tarball_url')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=2&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_1_0_0[a-z].*"))][0] | .tarball_url')" \ + "$(curl --silent "https://api.github.com/repos/openssl/openssl/tags?page=2&per_page=100" | jq '[.[] | select(.name|test("OpenSSL_0_9_8[a-z].*"))][0] | .tarball_url')" \ + "$(curl --silent "https://api.github.com/repos/libusb/libusb/releases/latest" | jq -r '.assets[1].browser_download_url')" \ + "$(echo 'https://salsa.debian.org/api/v4/projects/19181/repository/archive.tar.bz2?sha='$(curl --silent https://salsa.debian.org/api/v4/projects/19181/repository/tags | jq -r '.[0].commit.id'))" \ + "$(curl --silent https://api.github.com/repos/madler/zlib/tags | jq -r '.[0].tarball_url')" \ ); sample_tasks=( '( "(\"export TOOLCHAIN=@TOOLCHAIN@;\")" "(\"export CC=@CC@;\")" "(\"export RANLIB=@RANLIB@;\")" "('"'"'export PATH=\"\$TOOLCHAIN/bin:\$PATH\";'"'"')" "(\"./Configure --prefix=@PREFIX@/ linux-generic32 shared;\")" "(\"make;\")" "(\"make install;\")" )' \ '( "(\"export TOOLCHAIN=@TOOLCHAIN@;\")" "(\"export CC=@CC@;\")" "(\"export RANLIB=@RANLIB@;\")" "('"'"'export PATH=\"\$TOOLCHAIN/bin:\$PATH\";'"'"')" "(\"./Configure --prefix=@PREFIX@/ linux-generic32 shared;\")" "(\"make;\")" "(\"make install;\")" )' \ @@ -376,7 +453,7 @@ _create_config(){ declare -a tasks="${sample_tasks[$i]}" for t in "${tasks[@]}"; do - echo -e "$key"$([ ! -z $setting ] && echo "_")"$setting"$([ "$j" -gt "0" ] && echo "+")"=$t; $([ "$i" -eq "0" ] && echo "${sample_comments[5]}$(($j+1))")" >>"$configname"; + echo -e "$key"$([ ! -z $setting ] && echo "_")"$setting"$([ "$j" -gt "0" ] && echo "+")"=$t;$([ "$i" -eq "0" ] && echo " ${sample_comments[5]}$(($j+1))")" >>"$configname"; ((j++)); done;; esac;