Merge pull request #1597 from MilhouseVH/package_verify

build system: verify package downloads using sha256 checksum

config/path

@@ -82,6 +82,7 @@ unset LD_LIBRARY_PATH
   PKG_IS_ADDON="no"
   PKG_PATCH_DIRS=""
   PKG_NEED_UNPACK=""
+  PKG_SHA256=""
 
   if [ -n "$1" ]; then
     _PKG_ROOT_NAME=${1%:*}

scripts/get

@@ -1,25 +1,34 @@
 #!/bin/bash
 
 ################################################################################
-#      This file is part of OpenELEC - http://www.openelec.tv
+#      This file is part of LibreELEC - https://libreelec.tv
-#      Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv)
+#      Copyright (C) 2017-present Team LibreELEC
 #
-#  OpenELEC is free software: you can redistribute it and/or modify
+#  LibreELEC is free software: you can redistribute it and/or modify
 #  it under the terms of the GNU General Public License as published by
 #  the Free Software Foundation, either version 2 of the License, or
 #  (at your option) any later version.
 #
-#  OpenELEC is distributed in the hope that it will be useful,
+#  LibreELEC is distributed in the hope that it will be useful,
 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 #  GNU General Public License for more details.
 #
 #  You should have received a copy of the GNU General Public License
-#  along with OpenELEC.  If not, see <http://www.gnu.org/licenses/>.
+#  along with LibreELEC.  If not, see <http://www.gnu.org/licenses/>.
 ################################################################################
 
 . config/options $1
 
+_get_file_already_downloaded() {
+  if [ -f $PACKAGE ]; then
+    if [ "$(cat $STAMP_URL 2>/dev/null)" == "${PKG_URL}" ]; then
+      [ -z "${PKG_SHA256}" -o "$(cat $STAMP_SHA 2>/dev/null)" == "${PKG_SHA256}" ] && return 0
+    fi
+  fi
+  return 1
+}
+
 if [ -z "$1" ]; then
   for i in `find packages/ -type f -name package.mk`; do
     GET_PKG=`grep ^PKG_NAME= $i | sed -e "s,\",,g" -e "s,PKG_NAME=,,"`
@@ -27,47 +36,62 @@ if [ -z "$1" ]; then
   done
 fi
 
-if [ -n "$PKG_URL" -a -n "$PKG_SOURCE_NAME" ]; then
+[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 0
-  mkdir -p $SOURCES/$1
 
-  PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
+mkdir -p $SOURCES/$1
-  PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
-  [ "$VERBOSE" != "yes" ] && WGET_OPT=-q
-  WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $SOURCES/$1/$PKG_SOURCE_NAME"
 
-  STAMP="$PACKAGE.url"
+PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
+PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
+[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
+WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE"
 
-  # Nothing to be downloaded, exit now...
+STAMP_URL="$PACKAGE.url"
-  [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ] && exit 0
+STAMP_SHA="$PACKAGE.sha256"
 
-  # Avoid concurrent downloads of the same package
+# Latest file already present, exit now...
-  _isblocked=N
+_get_file_already_downloaded $1 && exit 0
-  exec 99<$SOURCES/$1
-  while ! flock --nonblock --exclusive 99; do
-    [ ${_isblocked} == N ] && { echo "Project ${PROJECT} waiting to avoid concurrent download of ${1}..."; _isblocked=Y; }
-    sleep 1
-  done
 
-  if ! [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ]; then
+# Avoid concurrent downloads of the same package
-    rm -f $SOURCES/$1/$PKG_SOURCE_NAME $STAMP
+_isblocked=N
+exec 99<$SOURCES/$1
+while ! flock --nonblock --exclusive 99; do
+  [ ${_isblocked} == N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent download of ${1}..."; _isblocked=Y; }
+  sleep 1
+done
 
-    printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor}      $1\n" ' '>&$SILENT_OUT
+# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
-    export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
+_get_file_already_downloaded $1 && exit 0
 
-    # unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
+# At this point, we need to download something...
-    unset LD_LIBRARY_PATH
+printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor}      $1\n" ' '>&$SILENT_OUT
+export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
 
-    NBWGET=1
+# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
-    until $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; do
+unset LD_LIBRARY_PATH
-      NBWGET=$((NBWGET + 1))
-      if [ $NBWGET -gt 10 ]; then
-        echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
-        exit 1
-      fi
-    done
 
-    echo "$PKG_URL" > $STAMP
+rm -f $STAMP_URL $STAMP_SHA
+
+NBWGET=10
+while [ $NBWGET -gt 0 ]; do
+  rm -f $PACKAGE
+
+  if $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; then
+    CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)"
+
+    [ -z "${PKG_SHA256}" -o "${PKG_SHA256}" == "${CALC_SHA256}" ] && break
+
+    printf "${boldred}WARNING${endcolor} Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n"
   fi
+  NBWGET=$((NBWGET - 1))
+done
+
+if [ $NBWGET -eq 0 ]; then
+  echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
+  exit 1
+else
+  printf "${boldgreen}INFO${endcolor} Calculated checksum is: ${CALC_SHA256}\n\n"
+  echo "${PKG_URL}" > $STAMP_URL
+  echo "${CALC_SHA256}" > $STAMP_SHA
 fi
 
 exit 0

tools/distro-tool

@@ -46,7 +46,7 @@ WORKER_MAX=${WORKER_MAX:-$(grep "^processor[[:space:]]*:" /proc/cpuinfo | wc -l)
 
 PYTHON_PROG='
 from __future__ import print_function
-import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil
+import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil, hashlib
 
 if sys.version_info >= (3, 0):
   import queue as Queue
@@ -125,7 +125,7 @@ class MyUtility(object):
   @staticmethod
   def readfile(filename):
     inputfile = codecs.open(filename, "rb", encoding="utf-8")
-    data= inputfile.read()
+    data = inputfile.read()
     inputfile.close()
     return data
 
@@ -320,6 +320,31 @@ class MyUtility(object):
 
     return result
 
+  # Calculate hash for chunked data
+  @staticmethod
+  def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):
+    for block in bytesiter:
+      hasher.update(block)
+    return (hasher.hexdigest() if ashexstr else hasher.digest())
+
+  # Read file in blocks/chunks to be memory efficient
+  @staticmethod
+  def file_as_blockiter(afile, blocksize=65536):
+    with afile:
+      block = afile.read(blocksize)
+      while len(block) > 0:
+        yield block
+        block = afile.read(blocksize)
+
+  # Calculate sha256 hash for a file
+  @staticmethod
+  def calculate_sha256(fname):
+    try:
+      return MyUtility.hash_bytestr_iter(MyUtility.file_as_blockiter(open(fname, "rb")), hashlib.sha256())
+    except:
+      raise
+      return ""
+
   # Use wget with same parameters as scripts/get is using
   @staticmethod
   def download_file(msgs, filename_data, filename_log, url):
@@ -336,7 +361,7 @@ class MyUtility(object):
     return False
 
   @staticmethod
-  def get_package(msgs, package_name, package_source, package_url):
+  def get_package(msgs, package_name, package_source, package_url, package_sha):
     onsource = False
     onmirror = False
 
@@ -384,6 +409,14 @@ class MyUtility(object):
       if os.path.exists(tmpfile_log):
         MyUtility.logmsg(msgs, 0, MyUtility.readfile(tmpfile_log))
     else:
+      if package_sha:
+        calc_sha = MyUtility.calculate_sha256(tmpfile_data)
+        if calc_sha != package_sha:
+          result = False
+          MyUtility.show(msgs, 0, "red", "DOWNLOAD FAILED!!", "%s (%s)" % (package_name, package_url))
+          MyUtility.logmsg(msgs, 0, "Checksum mismatch - got [%s], wanted [%s]" % (calc_sha, package_sha))
+
+    if result == True:
       MyUtility.show(msgs, 0, "green", "Successful Download", "%s (%s)" % (package_name, package_source))
       if IS_MIRROR:
         if not os.path.exists("%s/%s" % (DOWNLOAD_DIR, package_name)):
@@ -483,6 +516,7 @@ class MyThread(threading.Thread):
         pkg_name = qItem["PKG_NAME"]
         pkg_version = qItem["PKG_VERSION"]
         pkg_url = qItem["PKG_URL"]
+        pkg_sha = qItem["PKG_SHA256"]
         pkg_section = qItem["PKG_SECTION"]
         pkg_source_name = qItem["PKG_SOURCE_NAME"]
 
@@ -499,14 +533,14 @@ class MyThread(threading.Thread):
 
         self.output_queue.put([{"start": True, "name": threading.current_thread().name, "data": {"url": pkg_url, "tstamp": datetime.datetime.now()}}])
 
-        MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s" % (pkg_name, pkg_version, pkg_url))
+        MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s, wanted sha256 %s" % (pkg_name, pkg_version, pkg_url, pkg_sha))
 
         if MyUtility.have_package(pkg_name, pkg_source_name):
           MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name))
         else:
           tStart = datetime.datetime.now()
           if not stopped.is_set() and \
-             not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url):
+             not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url, pkg_sha):
              if not IGNORE_ERRORS:
                stopped.set()
           tDelta_get_package = datetime.datetime.now() - tStart
@@ -758,7 +792,7 @@ generate_work_worker() {
   local pcount=$1 worker="$2" revision="$3"
   local workfile_i="$(printf "%s.%02d" "${WORKFILES_I}" ${worker})"
   local workfile_o="$(printf "%s.%02d" "${WORKFILES_O}" ${worker})"
-  local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
+  local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SHA256 PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
   local package_name var comma PKG_URL PKG_SOURCE_NAME PKG_VERSION PKG_IS_ADDON
 
   [ -f "${workfile_i}" ] || return 0