diff --git a/config/path b/config/path
index 3e02ab401d..24e823e107 100644
--- a/config/path
+++ b/config/path
@@ -82,6 +82,7 @@ unset LD_LIBRARY_PATH
PKG_IS_ADDON="no"
PKG_PATCH_DIRS=""
PKG_NEED_UNPACK=""
+ PKG_SHA256=""
if [ -n "$1" ]; then
_PKG_ROOT_NAME=${1%:*}
diff --git a/scripts/get b/scripts/get
index 9199c38bea..5bd51b1cff 100755
--- a/scripts/get
+++ b/scripts/get
@@ -1,25 +1,34 @@
#!/bin/bash
################################################################################
-# This file is part of OpenELEC - http://www.openelec.tv
-# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv)
+# This file is part of LibreELEC - https://libreelec.tv
+# Copyright (C) 2017-present Team LibreELEC
#
-# OpenELEC is free software: you can redistribute it and/or modify
+# LibreELEC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
#
-# OpenELEC is distributed in the hope that it will be useful,
+# LibreELEC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
-# along with OpenELEC. If not, see .
+# along with LibreELEC. If not, see .
################################################################################
. config/options $1
+_get_file_already_downloaded() {
+ if [ -f $PACKAGE ]; then
+ if [ "$(cat $STAMP_URL 2>/dev/null)" == "${PKG_URL}" ]; then
+ [ -z "${PKG_SHA256}" -o "$(cat $STAMP_SHA 2>/dev/null)" == "${PKG_SHA256}" ] && return 0
+ fi
+ fi
+ return 1
+}
+
if [ -z "$1" ]; then
for i in `find packages/ -type f -name package.mk`; do
GET_PKG=`grep ^PKG_NAME= $i | sed -e "s,\",,g" -e "s,PKG_NAME=,,"`
@@ -27,47 +36,62 @@ if [ -z "$1" ]; then
done
fi
-if [ -n "$PKG_URL" -a -n "$PKG_SOURCE_NAME" ]; then
- mkdir -p $SOURCES/$1
+[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 0
- PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
- PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
- [ "$VERBOSE" != "yes" ] && WGET_OPT=-q
- WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $SOURCES/$1/$PKG_SOURCE_NAME"
+mkdir -p $SOURCES/$1
- STAMP="$PACKAGE.url"
+PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
+PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
+[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
+WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE"
- # Nothing to be downloaded, exit now...
- [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ] && exit 0
+STAMP_URL="$PACKAGE.url"
+STAMP_SHA="$PACKAGE.sha256"
- # Avoid concurrent downloads of the same package
- _isblocked=N
- exec 99<$SOURCES/$1
- while ! flock --nonblock --exclusive 99; do
- [ ${_isblocked} == N ] && { echo "Project ${PROJECT} waiting to avoid concurrent download of ${1}..."; _isblocked=Y; }
- sleep 1
- done
+# Latest file already present, exit now...
+_get_file_already_downloaded $1 && exit 0
- if ! [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ]; then
- rm -f $SOURCES/$1/$PKG_SOURCE_NAME $STAMP
+# Avoid concurrent downloads of the same package
+_isblocked=N
+exec 99<$SOURCES/$1
+while ! flock --nonblock --exclusive 99; do
+ [ ${_isblocked} == N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent download of ${1}..."; _isblocked=Y; }
+ sleep 1
+done
- printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT
- export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
+# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
+_get_file_already_downloaded $1 && exit 0
- # unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
- unset LD_LIBRARY_PATH
+# At this point, we need to download something...
+printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT
+export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
- NBWGET=1
- until $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; do
- NBWGET=$((NBWGET + 1))
- if [ $NBWGET -gt 10 ]; then
- echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
- exit 1
- fi
- done
+# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
+unset LD_LIBRARY_PATH
- echo "$PKG_URL" > $STAMP
+rm -f $STAMP_URL $STAMP_SHA
+
+NBWGET=10
+while [ $NBWGET -gt 0 ]; do
+ rm -f $PACKAGE
+
+ if $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; then
+ CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)"
+
+ [ -z "${PKG_SHA256}" -o "${PKG_SHA256}" == "${CALC_SHA256}" ] && break
+
+ printf "${boldred}WARNING${endcolor} Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n"
fi
+ NBWGET=$((NBWGET - 1))
+done
+
+if [ $NBWGET -eq 0 ]; then
+ echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
+ exit 1
+else
+ printf "${boldgreen}INFO${endcolor} Calculated checksum is: ${CALC_SHA256}\n\n"
+ echo "${PKG_URL}" > $STAMP_URL
+ echo "${CALC_SHA256}" > $STAMP_SHA
fi
exit 0
diff --git a/tools/distro-tool b/tools/distro-tool
index eecb1c5f01..1a0c176abc 100755
--- a/tools/distro-tool
+++ b/tools/distro-tool
@@ -46,7 +46,7 @@ WORKER_MAX=${WORKER_MAX:-$(grep "^processor[[:space:]]*:" /proc/cpuinfo | wc -l)
PYTHON_PROG='
from __future__ import print_function
-import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil
+import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil, hashlib
if sys.version_info >= (3, 0):
import queue as Queue
@@ -125,7 +125,7 @@ class MyUtility(object):
@staticmethod
def readfile(filename):
inputfile = codecs.open(filename, "rb", encoding="utf-8")
- data= inputfile.read()
+ data = inputfile.read()
inputfile.close()
return data
@@ -320,6 +320,31 @@ class MyUtility(object):
return result
+ # Calculate hash for chunked data
+ @staticmethod
+ def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):
+ for block in bytesiter:
+ hasher.update(block)
+ return (hasher.hexdigest() if ashexstr else hasher.digest())
+
+ # Read file in blocks/chunks to be memory efficient
+ @staticmethod
+ def file_as_blockiter(afile, blocksize=65536):
+ with afile:
+ block = afile.read(blocksize)
+ while len(block) > 0:
+ yield block
+ block = afile.read(blocksize)
+
+ # Calculate sha256 hash for a file
+ @staticmethod
+ def calculate_sha256(fname):
+ try:
+ return MyUtility.hash_bytestr_iter(MyUtility.file_as_blockiter(open(fname, "rb")), hashlib.sha256())
+ except:
+ raise
+ return ""
+
# Use wget with same parameters as scripts/get is using
@staticmethod
def download_file(msgs, filename_data, filename_log, url):
@@ -336,7 +361,7 @@ class MyUtility(object):
return False
@staticmethod
- def get_package(msgs, package_name, package_source, package_url):
+ def get_package(msgs, package_name, package_source, package_url, package_sha):
onsource = False
onmirror = False
@@ -384,6 +409,14 @@ class MyUtility(object):
if os.path.exists(tmpfile_log):
MyUtility.logmsg(msgs, 0, MyUtility.readfile(tmpfile_log))
else:
+ if package_sha:
+ calc_sha = MyUtility.calculate_sha256(tmpfile_data)
+ if calc_sha != package_sha:
+ result = False
+ MyUtility.show(msgs, 0, "red", "DOWNLOAD FAILED!!", "%s (%s)" % (package_name, package_url))
+ MyUtility.logmsg(msgs, 0, "Checksum mismatch - got [%s], wanted [%s]" % (calc_sha, package_sha))
+
+ if result == True:
MyUtility.show(msgs, 0, "green", "Successful Download", "%s (%s)" % (package_name, package_source))
if IS_MIRROR:
if not os.path.exists("%s/%s" % (DOWNLOAD_DIR, package_name)):
@@ -483,6 +516,7 @@ class MyThread(threading.Thread):
pkg_name = qItem["PKG_NAME"]
pkg_version = qItem["PKG_VERSION"]
pkg_url = qItem["PKG_URL"]
+ pkg_sha = qItem["PKG_SHA256"]
pkg_section = qItem["PKG_SECTION"]
pkg_source_name = qItem["PKG_SOURCE_NAME"]
@@ -499,14 +533,14 @@ class MyThread(threading.Thread):
self.output_queue.put([{"start": True, "name": threading.current_thread().name, "data": {"url": pkg_url, "tstamp": datetime.datetime.now()}}])
- MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s" % (pkg_name, pkg_version, pkg_url))
+ MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s, wanted sha256 %s" % (pkg_name, pkg_version, pkg_url, pkg_sha))
if MyUtility.have_package(pkg_name, pkg_source_name):
MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name))
else:
tStart = datetime.datetime.now()
if not stopped.is_set() and \
- not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url):
+ not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url, pkg_sha):
if not IGNORE_ERRORS:
stopped.set()
tDelta_get_package = datetime.datetime.now() - tStart
@@ -758,7 +792,7 @@ generate_work_worker() {
local pcount=$1 worker="$2" revision="$3"
local workfile_i="$(printf "%s.%02d" "${WORKFILES_I}" ${worker})"
local workfile_o="$(printf "%s.%02d" "${WORKFILES_O}" ${worker})"
- local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
+ local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SHA256 PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
local package_name var comma PKG_URL PKG_SOURCE_NAME PKG_VERSION PKG_IS_ADDON
[ -f "${workfile_i}" ] || return 0