mirror of
https://github.com/LibreELEC/LibreELEC.tv.git
synced 2025-07-28 21:26:49 +00:00
Merge pull request #1597 from MilhouseVH/package_verify
build system: verify package downloads using sha256 checksum
This commit is contained in:
Christian Hewitt
GitHub
commit
099fdf2cbf
@ -82,6 +82,7 @@ unset LD_LIBRARY_PATH
|
||||
PKG_IS_ADDON="no"
|
||||
PKG_PATCH_DIRS=""
|
||||
PKG_NEED_UNPACK=""
|
||||
PKG_SHA256=""
|
||||
|
||||
if [ -n "$1" ]; then
|
||||
_PKG_ROOT_NAME=${1%:*}
|
||||
|
||||
96
scripts/get
96
scripts/get
@ -1,25 +1,34 @@
|
||||
#!/bin/bash
|
||||
|
||||
################################################################################
|
||||
# This file is part of OpenELEC - http://www.openelec.tv
|
||||
# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv)
|
||||
# This file is part of LibreELEC - https://libreelec.tv
|
||||
# Copyright (C) 2017-present Team LibreELEC
|
||||
#
|
||||
# OpenELEC is free software: you can redistribute it and/or modify
|
||||
# LibreELEC is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU General Public License as published by
|
||||
# the Free Software Foundation, either version 2 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# OpenELEC is distributed in the hope that it will be useful,
|
||||
# LibreELEC is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with OpenELEC. If not, see <http://www.gnu.org/licenses/>.
|
||||
# along with LibreELEC. If not, see <http://www.gnu.org/licenses/>.
|
||||
################################################################################
|
||||
|
||||
. config/options $1
|
||||
|
||||
_get_file_already_downloaded() {
|
||||
if [ -f $PACKAGE ]; then
|
||||
if [ "$(cat $STAMP_URL 2>/dev/null)" == "${PKG_URL}" ]; then
|
||||
[ -z "${PKG_SHA256}" -o "$(cat $STAMP_SHA 2>/dev/null)" == "${PKG_SHA256}" ] && return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
if [ -z "$1" ]; then
|
||||
for i in `find packages/ -type f -name package.mk`; do
|
||||
GET_PKG=`grep ^PKG_NAME= $i | sed -e "s,\",,g" -e "s,PKG_NAME=,,"`
|
||||
@ -27,47 +36,62 @@ if [ -z "$1" ]; then
|
||||
done
|
||||
fi
|
||||
|
||||
if [ -n "$PKG_URL" -a -n "$PKG_SOURCE_NAME" ]; then
|
||||
mkdir -p $SOURCES/$1
|
||||
[ -z "$PKG_URL" -o -z "$PKG_SOURCE_NAME" ] && exit 0
|
||||
|
||||
PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
|
||||
PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
|
||||
[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
|
||||
WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $SOURCES/$1/$PKG_SOURCE_NAME"
|
||||
mkdir -p $SOURCES/$1
|
||||
|
||||
STAMP="$PACKAGE.url"
|
||||
PACKAGE="$SOURCES/$1/$PKG_SOURCE_NAME"
|
||||
PACKAGE_MIRROR="$DISTRO_MIRROR/$PKG_NAME/$PKG_SOURCE_NAME"
|
||||
[ "$VERBOSE" != "yes" ] && WGET_OPT=-q
|
||||
WGET_CMD="wget --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c $WGET_OPT -O $PACKAGE"
|
||||
|
||||
# Nothing to be downloaded, exit now...
|
||||
[ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ] && exit 0
|
||||
STAMP_URL="$PACKAGE.url"
|
||||
STAMP_SHA="$PACKAGE.sha256"
|
||||
|
||||
# Avoid concurrent downloads of the same package
|
||||
_isblocked=N
|
||||
exec 99<$SOURCES/$1
|
||||
while ! flock --nonblock --exclusive 99; do
|
||||
[ ${_isblocked} == N ] && { echo "Project ${PROJECT} waiting to avoid concurrent download of ${1}..."; _isblocked=Y; }
|
||||
sleep 1
|
||||
done
|
||||
# Latest file already present, exit now...
|
||||
_get_file_already_downloaded $1 && exit 0
|
||||
|
||||
if ! [ -f $SOURCES/$1/$PKG_SOURCE_NAME -a "$(cat $STAMP 2>/dev/null)" == "$PKG_URL" ]; then
|
||||
rm -f $SOURCES/$1/$PKG_SOURCE_NAME $STAMP
|
||||
# Avoid concurrent downloads of the same package
|
||||
_isblocked=N
|
||||
exec 99<$SOURCES/$1
|
||||
while ! flock --nonblock --exclusive 99; do
|
||||
[ ${_isblocked} == N ] && { echo "Project/Device ${DEVICE:-${PROJECT}} waiting, to avoid concurrent download of ${1}..."; _isblocked=Y; }
|
||||
sleep 1
|
||||
done
|
||||
|
||||
printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT
|
||||
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
|
||||
# Check again in case of concurrent access - if nothing needs to be downloaded, exit now...
|
||||
_get_file_already_downloaded $1 && exit 0
|
||||
|
||||
# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
|
||||
unset LD_LIBRARY_PATH
|
||||
# At this point, we need to download something...
|
||||
printf "%${BUILD_INDENT}c ${boldcyan}GET${endcolor} $1\n" ' '>&$SILENT_OUT
|
||||
export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE))
|
||||
|
||||
NBWGET=1
|
||||
until $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; do
|
||||
NBWGET=$((NBWGET + 1))
|
||||
if [ $NBWGET -gt 10 ]; then
|
||||
echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# unset LD_LIBRARY_PATH to stop wget from using toolchain/lib and loading libssl.so/libcrypto.so instead of host libraries
|
||||
unset LD_LIBRARY_PATH
|
||||
|
||||
echo "$PKG_URL" > $STAMP
|
||||
rm -f $STAMP_URL $STAMP_SHA
|
||||
|
||||
NBWGET=10
|
||||
while [ $NBWGET -gt 0 ]; do
|
||||
rm -f $PACKAGE
|
||||
|
||||
if $WGET_CMD "$PKG_URL" || $WGET_CMD "$PACKAGE_MIRROR"; then
|
||||
CALC_SHA256="$(sha256sum $PACKAGE | cut -d" " -f1)"
|
||||
|
||||
[ -z "${PKG_SHA256}" -o "${PKG_SHA256}" == "${CALC_SHA256}" ] && break
|
||||
|
||||
printf "${boldred}WARNING${endcolor} Incorrect checksum calculated on downloaded file: got ${CALC_SHA256}, wanted ${PKG_SHA256}\n\n"
|
||||
fi
|
||||
NBWGET=$((NBWGET - 1))
|
||||
done
|
||||
|
||||
if [ $NBWGET -eq 0 ]; then
|
||||
echo -e "\nCant't get $1 sources : $PKG_URL\n Try later !!"
|
||||
exit 1
|
||||
else
|
||||
printf "${boldgreen}INFO${endcolor} Calculated checksum is: ${CALC_SHA256}\n\n"
|
||||
echo "${PKG_URL}" > $STAMP_URL
|
||||
echo "${CALC_SHA256}" > $STAMP_SHA
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
@ -46,7 +46,7 @@ WORKER_MAX=${WORKER_MAX:-$(grep "^processor[[:space:]]*:" /proc/cpuinfo | wc -l)
|
||||
|
||||
PYTHON_PROG='
|
||||
from __future__ import print_function
|
||||
import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil
|
||||
import sys, os, json, codecs, re, threading, subprocess, glob, datetime, shutil, hashlib
|
||||
|
||||
if sys.version_info >= (3, 0):
|
||||
import queue as Queue
|
||||
@ -125,7 +125,7 @@ class MyUtility(object):
|
||||
@staticmethod
|
||||
def readfile(filename):
|
||||
inputfile = codecs.open(filename, "rb", encoding="utf-8")
|
||||
data= inputfile.read()
|
||||
data = inputfile.read()
|
||||
inputfile.close()
|
||||
return data
|
||||
|
||||
@ -320,6 +320,31 @@ class MyUtility(object):
|
||||
|
||||
return result
|
||||
|
||||
# Calculate hash for chunked data
|
||||
@staticmethod
|
||||
def hash_bytestr_iter(bytesiter, hasher, ashexstr=True):
|
||||
for block in bytesiter:
|
||||
hasher.update(block)
|
||||
return (hasher.hexdigest() if ashexstr else hasher.digest())
|
||||
|
||||
# Read file in blocks/chunks to be memory efficient
|
||||
@staticmethod
|
||||
def file_as_blockiter(afile, blocksize=65536):
|
||||
with afile:
|
||||
block = afile.read(blocksize)
|
||||
while len(block) > 0:
|
||||
yield block
|
||||
block = afile.read(blocksize)
|
||||
|
||||
# Calculate sha256 hash for a file
|
||||
@staticmethod
|
||||
def calculate_sha256(fname):
|
||||
try:
|
||||
return MyUtility.hash_bytestr_iter(MyUtility.file_as_blockiter(open(fname, "rb")), hashlib.sha256())
|
||||
except:
|
||||
raise
|
||||
return ""
|
||||
|
||||
# Use wget with same parameters as scripts/get is using
|
||||
@staticmethod
|
||||
def download_file(msgs, filename_data, filename_log, url):
|
||||
@ -336,7 +361,7 @@ class MyUtility(object):
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_package(msgs, package_name, package_source, package_url):
|
||||
def get_package(msgs, package_name, package_source, package_url, package_sha):
|
||||
onsource = False
|
||||
onmirror = False
|
||||
|
||||
@ -384,6 +409,14 @@ class MyUtility(object):
|
||||
if os.path.exists(tmpfile_log):
|
||||
MyUtility.logmsg(msgs, 0, MyUtility.readfile(tmpfile_log))
|
||||
else:
|
||||
if package_sha:
|
||||
calc_sha = MyUtility.calculate_sha256(tmpfile_data)
|
||||
if calc_sha != package_sha:
|
||||
result = False
|
||||
MyUtility.show(msgs, 0, "red", "DOWNLOAD FAILED!!", "%s (%s)" % (package_name, package_url))
|
||||
MyUtility.logmsg(msgs, 0, "Checksum mismatch - got [%s], wanted [%s]" % (calc_sha, package_sha))
|
||||
|
||||
if result == True:
|
||||
MyUtility.show(msgs, 0, "green", "Successful Download", "%s (%s)" % (package_name, package_source))
|
||||
if IS_MIRROR:
|
||||
if not os.path.exists("%s/%s" % (DOWNLOAD_DIR, package_name)):
|
||||
@ -483,6 +516,7 @@ class MyThread(threading.Thread):
|
||||
pkg_name = qItem["PKG_NAME"]
|
||||
pkg_version = qItem["PKG_VERSION"]
|
||||
pkg_url = qItem["PKG_URL"]
|
||||
pkg_sha = qItem["PKG_SHA256"]
|
||||
pkg_section = qItem["PKG_SECTION"]
|
||||
pkg_source_name = qItem["PKG_SOURCE_NAME"]
|
||||
|
||||
@ -499,14 +533,14 @@ class MyThread(threading.Thread):
|
||||
|
||||
self.output_queue.put([{"start": True, "name": threading.current_thread().name, "data": {"url": pkg_url, "tstamp": datetime.datetime.now()}}])
|
||||
|
||||
MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s" % (pkg_name, pkg_version, pkg_url))
|
||||
MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s, wanted sha256 %s" % (pkg_name, pkg_version, pkg_url, pkg_sha))
|
||||
|
||||
if MyUtility.have_package(pkg_name, pkg_source_name):
|
||||
MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name))
|
||||
else:
|
||||
tStart = datetime.datetime.now()
|
||||
if not stopped.is_set() and \
|
||||
not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url):
|
||||
not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url, pkg_sha):
|
||||
if not IGNORE_ERRORS:
|
||||
stopped.set()
|
||||
tDelta_get_package = datetime.datetime.now() - tStart
|
||||
@ -758,7 +792,7 @@ generate_work_worker() {
|
||||
local pcount=$1 worker="$2" revision="$3"
|
||||
local workfile_i="$(printf "%s.%02d" "${WORKFILES_I}" ${worker})"
|
||||
local workfile_o="$(printf "%s.%02d" "${WORKFILES_O}" ${worker})"
|
||||
local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
|
||||
local wanted_vars="PKG_NAME PKG_VERSION PKG_URL PKG_SHA256 PKG_SECTION PKG_IS_ADDON PKG_SOURCE_NAME"
|
||||
local package_name var comma PKG_URL PKG_SOURCE_NAME PKG_VERSION PKG_IS_ADDON
|
||||
|
||||
[ -f "${workfile_i}" ] || return 0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user