diff --git a/config/path b/config/path index 3e02ab401d..24e823e107 100644 --- a/config/path +++ b/config/path @@ -82,6 +82,7 @@ unset LD_LIBRARY_PATH PKG_IS_ADDON="no" PKG_PATCH_DIRS="" PKG_NEED_UNPACK="" + PKG_SHA256="" if [ -n "$1" ]; then _PKG_ROOT_NAME=${1%:*} diff --git a/scripts/get b/scripts/get index 9199c38bea..5bd51b1cff 100755 --- a/scripts/get +++ b/scripts/get @@ -1,25 +1,34 @@ #!/bin/bash ################################################################################ -# This file is part of OpenELEC - http://www.openelec.tv -# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# This file is part of LibreELEC - https://libreelec.tv +# Copyright (C) 2017-present Team LibreELEC # -# OpenELEC is free software: you can redistribute it and/or modify +# LibreELEC is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 2 of the License, or # (at your option) any later version. # -# OpenELEC is distributed in the hope that it will be useful, +# LibreELEC is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with OpenELEC. If not, see . +# along with LibreELEC. If not, see . ################################################################################ . config/options $1 +_get_file_already_downloaded() { + if [ -f $PACKAGE ]; then + if [ "$(cat $STAMP_URL 2>/dev/null)" == "${PKG_URL}" ]; then + [ -z "${PKG_SHA256}" -o "$(cat $STAMP_SHA 2>/dev/null)" == "${PKG_SHA256}" ] && return 0 + fi + fi + return 1 +} + if [ -z "$1" ]; then for i in `find packages/ -type f -name package.mk`; do GET_PKG=`grep ^PKG_NAME= $i | sed -e "s,\",,g" -e "s,PKG_NAME=,,"` @@ -27,47 +36,62 @@ if [ -z "$1" ]; then done fi -if [ -n "$PKG_URL" -a -n "$PKG_SOURCE_NAME" ]; then - mkdir -p $SOURCES/$1 +[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 0 - PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME" - PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME" - [ "$VERBOSE" != "yes" ] && WGET_OPT=-q - WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $SOURCES/$1/$PKG_SOURCE_NAME" +mkdir -p $SOURCES/$1 - STAMP="$PACKAGE.url" +PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME" +PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME" +[ "$VERBOSE" != "yes" ] && WGET_OPT=-q +WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE" - # Nothing to be downloaded, exit now... - [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ] && exit 0 +STAMP_URL="$PACKAGE.url" +STAMP_SHA="$PACKAGE.sha256" - # Avoid concurrent downloads of the same package - _isblocked=N - exec 99<$SOURCES/$1 - while ! flock --nonblock --exclusive 99; do - [ ${_isblocked} == N ] && { echo "Project ${PROJECT} waiting to avoid concurrent download of ${1}..."; _isblocked=Y; } - sleep 1 - done +# Latest file already present, exit now... +_get_file_already_downloaded $1 && exit 0 - if ! [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ]; then - rm -f $SOURCES/$1/$PKG_SOURCE_NAME $STAMP +# Avoid concurrent downloads of the same package +_isblocked=N +exec 99<$SOURCES/$1 +while ! flock --nonblock --exclusive 99; do + [ ${_isblocked} == N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent download of ${1}..."; _isblocked=Y; } + sleep 1 +done - printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT - export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE)) +# Check again in case of concurrent access - if nothing needs to be downloaded, exit now... +_get_file_already_downloaded $1 && exit 0 - # unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries - unset LD_LIBRARY_PATH +# At this point, we need to download something... +printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT +export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE)) - NBWGET=1 - until $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; do - NBWGET=$((NBWGET + 1)) - if [ $NBWGET -gt 10 ]; then - echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!" - exit 1 - fi - done +# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries +unset LD_LIBRARY_PATH - echo "$PKG_URL" > $STAMP +rm -f $STAMP_URL $STAMP_SHA + +NBWGET=10 +while [ $NBWGET -gt 0 ]; do + rm -f $PACKAGE + + if $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; then + CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)" + + [ -z "${PKG_SHA256}" -o "${PKG_SHA256}" == "${CALC_SHA256}" ] && break + + printf "${boldred}WARNING${endcolor} Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n" fi + NBWGET=$((NBWGET - 1)) +done + +if [ $NBWGET -eq 0 ]; then + echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!" + exit 1 +else + printf "${boldgreen}INFO${endcolor} Calculated checksum is: ${CALC_SHA256}\n\n" + echo "${PKG_URL}" > $STAMP_URL + echo "${CALC_SHA256}" > $STAMP_SHA fi exit 0 diff --git a/tools/distro-tool b/tools/distro-tool index eecb1c5f01..1a0c176abc 100755 --- a/tools/distro-tool +++ b/tools/distro-tool @@ -46,7 +46,7 @@ WORKER_MAX=${WORKER_MAX:-$(grep "^processor[[:space:]]*:" /proc/cpuinfo | wc -l) PYTHON_PROG=' from __future__ import print_function -import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil +import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil, hashlib if sys.version_info >= (3, 0): import queue as Queue @@ -125,7 +125,7 @@ class MyUtility(object): @staticmethod def readfile(filename): inputfile = codecs.open(filename, "rb", encoding="utf-8") - data= inputfile.read() + data = inputfile.read() inputfile.close() return data @@ -320,6 +320,31 @@ class MyUtility(object): return result + # Calculate hash for chunked data + @staticmethod + def hash_bytestr_iter(bytesiter, hasher, ashexstr=True): + for block in bytesiter: + hasher.update(block) + return (hasher.hexdigest() if ashexstr else hasher.digest()) + + # Read file in blocks/chunks to be memory efficient + @staticmethod + def file_as_blockiter(afile, blocksize=65536): + with afile: + block = afile.read(blocksize) + while len(block) > 0: + yield block + block = afile.read(blocksize) + + # Calculate sha256 hash for a file + @staticmethod + def calculate_sha256(fname): + try: + return MyUtility.hash_bytestr_iter(MyUtility.file_as_blockiter(open(fname, "rb")), hashlib.sha256()) + except: + raise + return "" + # Use wget with same parameters as scripts/get is using @staticmethod def download_file(msgs, filename_data, filename_log, url): @@ -336,7 +361,7 @@ class MyUtility(object): return False @staticmethod - def get_package(msgs, package_name, package_source, package_url): + def get_package(msgs, package_name, package_source, package_url, package_sha): onsource = False onmirror = False @@ -384,6 +409,14 @@ class MyUtility(object): if os.path.exists(tmpfile_log): MyUtility.logmsg(msgs, 0, MyUtility.readfile(tmpfile_log)) else: + if package_sha: + calc_sha = MyUtility.calculate_sha256(tmpfile_data) + if calc_sha != package_sha: + result = False + MyUtility.show(msgs, 0, "red", "DOWNLOAD FAILED!!", "%s (%s)" % (package_name, package_url)) + MyUtility.logmsg(msgs, 0, "Checksum mismatch - got [%s], wanted [%s]" % (calc_sha, package_sha)) + + if result == True: MyUtility.show(msgs, 0, "green", "Successful Download", "%s (%s)" % (package_name, package_source)) if IS_MIRROR: if not os.path.exists("%s/%s" % (DOWNLOAD_DIR, package_name)): @@ -483,6 +516,7 @@ class MyThread(threading.Thread): pkg_name = qItem["PKG_NAME"] pkg_version = qItem["PKG_VERSION"] pkg_url = qItem["PKG_URL"] + pkg_sha = qItem["PKG_SHA256"] pkg_section = qItem["PKG_SECTION"] pkg_source_name = qItem["PKG_SOURCE_NAME"] @@ -499,14 +533,14 @@ class MyThread(threading.Thread): self.output_queue.put([{"start": True, "name": threading.current_thread().name, "data": {"url": pkg_url, "tstamp": datetime.datetime.now()}}]) - MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s" % (pkg_name, pkg_version, pkg_url)) + MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s, wanted sha256 %s" % (pkg_name, pkg_version, pkg_url, pkg_sha)) if MyUtility.have_package(pkg_name, pkg_source_name): MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name)) else: tStart = datetime.datetime.now() if not stopped.is_set() and \ - not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url): + not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url, pkg_sha): if not IGNORE_ERRORS: stopped.set() tDelta_get_package = datetime.datetime.now() - tStart @@ -758,7 +792,7 @@ generate_work_worker() { local pcount=$1 worker="$2" revision="$3" local workfile_i="$(printf "%s.%02d" "${WORKFILES_I}" ${worker})" local workfile_o="$(printf "%s.%02d" "${WORKFILES_O}" ${worker})" - local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME" + local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SHA256 PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME" local package_name var comma PKG_URL PKG_SOURCE_NAME PKG_VERSION PKG_IS_ADDON [ -f "${workfile_i}" ] || return 0