diff --git a/packages/compress/bzip2/package.mk b/packages/compress/bzip2/package.mk index 4a3db22736..c163e97f01 100644 --- a/packages/compress/bzip2/package.mk +++ b/packages/compress/bzip2/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv) PKG_NAME="bzip2" PKG_VERSION="1.0.6" @@ -7,7 +8,7 @@ PKG_SHA256="a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd" PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="http://www.bzip.org" -PKG_URL="http://www.bzip.org/$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.gz" +PKG_URL="$DISTRO_SRC/$PKG_NAME-$PKG_VERSION.tar.gz" PKG_DEPENDS_HOST="" PKG_DEPENDS_TARGET="toolchain" PKG_SECTION="compress" diff --git a/packages/graphics/libepoxy/package.mk b/packages/graphics/libepoxy/package.mk index 28f1d38d89..c1f1e2c4e4 100644 --- a/packages/graphics/libepoxy/package.mk +++ b/packages/graphics/libepoxy/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv) # libepoxy (actually) needs to be built shared, to avoid # (EE) Failed to load /usr/lib/xorg/modules/libglamoregl.so: @@ -7,8 +8,8 @@ # in Xorg.log PKG_NAME="libepoxy" -PKG_VERSION="1.4.3" -PHG_SHA256="0b808a06c9685a62fca34b680abb8bc7fb2fda074478e329b063c1f872b826f6" +PKG_VERSION="1.5.2" +PKG_SHA256="a9562386519eb3fd7f03209f279f697a8cba520d3c155d6e253c3e138beca7d8" PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="https://github.com/anholt/libepoxy" diff --git a/tools/distro-tool b/tools/distro-tool index 4ba9d504ef..0d43b96e76 100755 --- a/tools/distro-tool +++ b/tools/distro-tool @@ -21,7 +21,9 @@ CHECK_FILTER_CLASS=All PROGRESS=yes IS_MIRROR=yes VERBOSE=0 -WORKER_THREADS=32 +PATH_FILTER= +VERIFY_CHECKSUM=no +WORKER_THREADS=16 LOCKFILE=/tmp/distro_tool.lock WORKFILES_I=/tmp/distro_tool.in @@ -244,11 +246,14 @@ class MyUtility(object): # 35/56 as retries, could have several of these before a successful # request - limit total number of retries as ${retry} * 3. while retry > 0 and maxattempts > 0: + maxattempts -= 1 if stopped.is_set(): break ts_cmd = datetime.datetime.now() if HEAD_supported: + MyUtility.logmsg(msgs, 3, "%s --head --output - --url %s" % (curl_args, url)) (result, headers) = MyUtility.runcommand(msgs, "%s --head --output - --url %s" % (curl_args, url)) else: + MyUtility.logmsg(msgs, 3, "%s --range 0-1024 --output /dev/null --url %s" % (curl_args, url)) (result, headers) = MyUtility.runcommand(msgs, "%s --range 0-1024 --output /dev/null --url %s" % (curl_args, url)) search_obj = MyUtility.search_HTTP_CODE.search(headers) @@ -352,12 +357,9 @@ class MyUtility(object): @staticmethod def get_package(msgs, package_name, package_source, package_url, package_sha): - onsource = False - onmirror = False onsource = MyUtility.remote_file_exists(msgs, package_url) - if not onsource: - onmirror = MyUtility.remote_file_exists(msgs, "%s/%s/%s" % (DISTRO_MIRROR, package_name, package_source)) + onmirror = MyUtility.remote_file_exists(msgs, "%s/%s/%s" % (DISTRO_MIRROR, package_name, package_source)) # If the only PKG_URL source is the DISTRO_SRC server... if package_url.startswith(DISTRO_SOURCE): @@ -386,12 +388,29 @@ class MyUtility(object): if os.path.exists(tmpfile_log): os.remove(tmpfile_log) + if VERIFY_CHECKSUM == True and package_sha and onmirror: + result = MyUtility.download_file(msgs, tmpfile_data, tmpfile_log, "%s/%s/%s" % (DISTRO_MIRROR, package_name, package_source)) + calc_sha = MyUtility.calculate_sha256(tmpfile_data) + + if os.path.exists(tmpfile_data): + os.remove(tmpfile_data) + if os.path.exists(tmpfile_log): + os.remove(tmpfile_log) + + if calc_sha != package_sha: + MyUtility.show(msgs, 0, "yellow", "CHECKSUM MISMATCH FROM MIRROR - REFRESHING FROM SOURCE", "%s (%s)" % (package_name, package_url)) + MyUtility.logmsg(msgs, 0, "Checksum mismatch - got [%s], expected [%s]" % (calc_sha, package_sha)) + onmirror = False + else: + MyUtility.show(msgs, 1, "green", "Mirror checksum valid", "%s (%s)" % (package_name, package_source)) + return True + if onsource: result = MyUtility.download_file(msgs, tmpfile_data, tmpfile_log, package_url) elif onmirror: result = MyUtility.download_file(msgs, tmpfile_data, tmpfile_log, "%s/%s/%s" % (DISTRO_MIRROR, package_name, package_source)) else: - result=False + result = False if result == False or not os.path.exists(tmpfile_data): result = False @@ -525,15 +544,15 @@ class MyThread(threading.Thread): MyUtility.logmsg(msgs, 3, ">>>>>>>>>>>>>>>>> %s, %s, %s, wanted sha256 %s" % (pkg_name, pkg_version, pkg_url, pkg_sha)) - if MyUtility.have_package(pkg_name, pkg_source_name): - MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name)) - else: + if not MyUtility.have_package(pkg_name, pkg_source_name) or (VERIFY_CHECKSUM and pkg_sha): tStart = datetime.datetime.now() if not stopped.is_set() and \ not MyUtility.get_package(msgs, pkg_name, pkg_source_name, pkg_url, pkg_sha): if not IGNORE_ERRORS: stopped.set() tDelta_get_package = datetime.datetime.now() - tStart + else: + MyUtility.show(msgs, 1, "green", "Already downloaded", "%s (%s)" % (pkg_name, pkg_source_name)) if CHECK_NEWER and not stopped.is_set(): if (CHECK_FILTER_CLASS == "All") or \ @@ -655,7 +674,8 @@ if __name__ == "__main__": GIT_PASSWORD = args[11] CHECK_FILTER_CLASS = args[12] CHECK_FILTER_TYPE = args[13] - VERBOSE = int(args[14]) + VERIFY_CHECKSUM = True if args[14] == "yes" else False + VERBOSE = int(args[15]) SCRATCH_DIR="%s/.tmp" % DOWNLOAD_DIR stopped = threading.Event() @@ -677,30 +697,32 @@ Usage: $(basename $0) -d|--download [-t|--target ] [-l|--libreelec [-m|--mirror] [-s|--source] [-a|-all] [-p|--package [-r|--revision ]] [--git] [-n|--notnewer] [--check-main | --check-addons] [--check-ver | --check-rev] [--noprogress] [-T #|--threads #] [-U|--gituser] [-P|--gitpass] [--dry-run] - [-v|--verbose] [-h|--help] + [--path-filter path] [--verify-checksum] [-v|--verbose] [-h|--help] Options: - -d, --download: Directory path into which new package files will be downloaded - default is $HOME/download[1] - -t, --target: Directory path for existing packages that are to be refreshed, default is $HOME/sources[2] - -l, --libreelec: LibreELEC.tv repo, default is ${HOME}/LibreELEC.tv - -m, --mirror: Target is mirror not source - mirror uses a hierarchical per-package folder structure - -s, --source: Target is source not mirror - source uses a flattened file structure. Default is mirror - -a, --all: Ignore download failures, continue processing all packages - -p, --package: Package to process, otherwise process all packages - -r, --revision: Version to use in place of PKG_VERSION, only applicable in conjunction with -p - --git: Clone (if not available) or pull the LibreELEC.tv repository - -n, --notnewer: Don't check for newer packages ('X.Y.Z+1' 'X.Y+1.0' 'X+1.0.0') - --check-main: Check newer for main packages, ignore add-on packages - --check-addons: Check newer for add-on packages, ignore main packages - --check-ver: Check newer for version-based packages, ignore rev-based/github packages - --check-rev: Check newer for rev-based/github packages, ignore version-based packages - --noprogress: Do not show progress indicator - -T, --threads: Number of worker threads, default is 32 - -U, --gituser: Git username (or source from ~/.git.conf) - avoids API limits - -P, --gitpass: Git password (or source from ~/.git.conf) - avoids API limits - --dry-run: Don't actually download anything (will still git clone/pull if configured) - -v, --verbose: Output more verbose sync information. Repeat for more detail - -h, --help: This message + -d, --download: Directory path into which new package files will be downloaded - default is $HOME/download[1] + -t, --target: Directory path for existing packages that are to be refreshed, default is $HOME/sources[2] + -l, --libreelec: LibreELEC.tv repo, default is ${HOME}/LibreELEC.tv + -m, --mirror: Target is mirror not source - mirror uses a hierarchical per-package folder structure + -s, --source: Target is source not mirror - source uses a flattened file structure. Default is mirror + -a, --all: Ignore download failures, continue processing all packages + -p, --package: Package to process, otherwise process all packages + -r, --revision: Version to use in place of PKG_VERSION, only applicable in conjunction with -p + --git: Clone (if not available) or pull the LibreELEC.tv repository + -n, --notnewer: Don't check for newer packages ('X.Y.Z+1' 'X.Y+1.0' 'X+1.0.0') + --check-main: Check newer for main packages, ignore add-on packages + --check-addons: Check newer for add-on packages, ignore main packages + --check-ver: Check newer for version-based packages, ignore rev-based/github packages + --check-rev: Check newer for rev-based/github packages, ignore version-based packages + --noprogress: Do not show progress indicator + -T, --threads: Number of worker threads, default is 32 + -U, --gituser: Git username (or source from ~/.git.conf) - avoids API limits + -P, --gitpass: Git password (or source from ~/.git.conf) - avoids API limits + --dry-run: Don't actually download anything (will still git clone/pull if configured) + --path-filter Restrict workload to specified path, eg. packages/x11 (recursive) + --verify-checksum: If checksum on mirror is no longer valid, attempt re-download of package from source + -v, --verbose: Output more verbose sync information. Repeat for more detail + -h, --help: This message Note#1. The download directory will have the LibreELEC version appended (eg. /devel) unless it is a mirror, in which case "/mirror" will be appended. Note#2. The target directory will have the LibreELEC version appended (eg. /devel) unless it is a mirror, in which case "/mirror" will be appended. @@ -721,18 +743,22 @@ get_libreelec_option() { } generate_work() { - local package_name="$1" revision="$2" + local package_name="$1" revision="$2" path_filter="$3" local packages pcount local workfile_o local tcount=0 [ ${PROGRESS} == yes ] && echo -en "Acquiring packages...\r" >&2 - packages="$(get_packages ${package_name})" + packages="$(get_packages "${package_name}" "${path_filter}")" pcount="$(echo "${packages}" | wc -l)" if [ -n "${package_name}" -a -z "${packages}" ]; then - echo "ERROR: ${package_name} is not a valid package - package.mk does not exist" >&2 + if [ -n "${path_filter}" ]; then + echo "ERROR: ${package_name} is not a valid package when using filter ${path_filter} - package.mk does not exist" >&2 + else + echo "ERROR: ${package_name} is not a valid package - package.mk does not exist" >&2 + fi exit 1 fi @@ -848,16 +874,24 @@ get_package_path() { } get_packages() { - local package_name="$1" + local package_name="$1" path_filter="$2" export -f get_package_path cd $LIBREELEC_DIR - if [ -n "${package_name}" ]; then - basename $(dirname $(find packages projects/${PROJECT} -path "*/${package_name}/*" -name package.mk) 2>/dev/null | head -1) 2>/dev/null + if [ -n "${path_filter}" ]; then + if [ -n "${package_name}" ]; then + basename $(dirname $(find ${path_filter} -path "*/${package_name}/*" -name package.mk 2>/dev/null) 2>/dev/null | head -1) 2>/dev/null + else + find ${path_filter} -name package.mk -exec bash -c get_package_path "{}" \; 2>/dev/null | cut -d' ' -f1 | sort -u + fi else - find packages projects/${PROJECT} -name package.mk -exec bash -c get_package_path "{}" \; | cut -d' ' -f1 | sort -u + if [ -n "${package_name}" ]; then + basename $(dirname $(find packages projects/${PROJECT} -path "*/${package_name}/*" -name package.mk 2>/dev/null) 2>/dev/null | head -1) 2>/dev/null + else + find packages projects/${PROJECT} -name package.mk -exec bash -c get_package_path "{}" \; 2>/dev/null | cut -d' ' -f1 | sort -u + fi fi return 0 } @@ -886,7 +920,7 @@ exec_worker_prog() { python3 /tmp/distro-tool.py "${DOWNLOAD_DIR}" "${TARGET_DIR}" "${DISTRO_SOURCE}" "${DISTRO_MIRROR}" \ "${IS_MIRROR}" "${IGNORE_ERRORS}" "${DRY_RUN}" "${CHECK_NEWER}" \ "${PROGRESS}" "${WORKER_THREADS}" "${GIT_USERNAME}" "${GIT_PASSWORD}" \ - "${CHECK_FILTER_CLASS}" "${CHECK_FILTER_TYPE}" "${VERBOSE}" + "${CHECK_FILTER_CLASS}" "${CHECK_FILTER_TYPE}" ${VERIFY_CHECKSUM} "${VERBOSE}" rm -f /tmp/distro-tool.py } @@ -962,6 +996,13 @@ while [ : ]; do --noprogress) PROGRESS=no ;; + --path-filter) + shift + PATH_FILTER=$1 + ;; + --verify-checksum) + VERIFY_CHECKSUM=yes + ;; -h|--help) help exit 0 @@ -1026,6 +1067,9 @@ echo "Distro Source is: ${DISTRO_SOURCE}" echo "Distro Mirror is: ${DISTRO_MIRROR}" echo " Syncing against: ${TARGET_DIR}" echo " Downloading to: ${DOWNLOAD_DIR}" +if [ -n "${PATH_FILTER}" ]; then + echo " Path filter: ${PATH_FILTER}" +fi echo " Check Newer: ${CHECK_NEWER^} Class/Type: ${CHECK_FILTER_CLASS} / ${CHECK_FILTER_TYPE}" echo " Dry run: ${DRY_RUN^}" if [ ${VERBOSE} -gt 2 ]; then @@ -1035,7 +1079,7 @@ fi echo if [ ${VERBOSE} -gt 2 -a -n "${DEBUG_LOG}" ]; then - generate_work "${PACKAGE}" "${REVISION}" | exec_worker_prog 2>${DEBUG_LOG} + generate_work "${PACKAGE}" "${REVISION}" "${PATH_FILTER}" | exec_worker_prog 2>${DEBUG_LOG} else - generate_work "${PACKAGE}" "${REVISION}" | exec_worker_prog + generate_work "${PACKAGE}" "${REVISION}" "${PATH_FILTER}" | exec_worker_prog fi