From 7bb5fefb42a582d9433e10febc3fc118532bf8cd Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 9 May 2019 01:53:52 +0100 Subject: [PATCH 01/14] buildsystem: addons install minor cleanup --- config/functions | 2 ++ packages/addons/service/slice/package.mk | 4 +--- scripts/install_addon | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/config/functions b/config/functions index 793ba089ca..af1020e884 100644 --- a/config/functions +++ b/config/functions @@ -1155,6 +1155,8 @@ done } install_addon_files() { + mkdir -p "$1" + install_addon_source "$1" install_addon_images "$1" create_addon_xml "$1" diff --git a/packages/addons/service/slice/package.mk b/packages/addons/service/slice/package.mk index 5fa2876837..c78baa5b30 100644 --- a/packages/addons/service/slice/package.mk +++ b/packages/addons/service/slice/package.mk @@ -20,9 +20,7 @@ PKG_ADDON_PROJECTS="Slice Slice3" PKG_ADDON_TYPE="xbmc.service" makeinstall_target() { - PKG_ADDON_INSTALL_DIR="${INSTALL}/usr/share/kodi/addons/${PKG_SECTION}.${PKG_NAME}" - mkdir -p "${PKG_ADDON_INSTALL_DIR}" - install_addon_files "${PKG_ADDON_INSTALL_DIR}" + install_addon_files "${INSTALL}/usr/share/kodi/addons/${PKG_SECTION}.${PKG_NAME}" } addon() { diff --git a/scripts/install_addon b/scripts/install_addon index 289b934b39..0a6220328a 100755 --- a/scripts/install_addon +++ b/scripts/install_addon @@ -12,8 +12,6 @@ setup_toolchain target ADDON_DIRECTORY="$ADDON_BUILD/$PKG_ADDON_ID" -mkdir -p "$ADDON_DIRECTORY" - install_addon_files "$ADDON_DIRECTORY" debug_strip "$ADDON_DIRECTORY" From 7d07f4b0f7a6fe95182eda49c49fcee654ada21e Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sun, 19 May 2019 23:25:32 +0100 Subject: [PATCH 02/14] linux: avoid building host and target concurrently --- packages/linux/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/linux/package.mk b/packages/linux/package.mk index 5d0a920ab6..a5c0dc4088 100644 --- a/packages/linux/package.mk +++ b/packages/linux/package.mk @@ -6,7 +6,7 @@ PKG_NAME="linux" PKG_LICENSE="GPL" PKG_SITE="http://www.kernel.org" PKG_DEPENDS_HOST="ccache:host openssl:host" -PKG_DEPENDS_TARGET="toolchain cpio:host kmod:host xz:host wireless-regdb keyutils $KERNEL_EXTRA_DEPENDS_TARGET" +PKG_DEPENDS_TARGET="toolchain linux:host cpio:host kmod:host xz:host wireless-regdb keyutils $KERNEL_EXTRA_DEPENDS_TARGET" PKG_DEPENDS_INIT="toolchain" PKG_NEED_UNPACK="$LINUX_DEPENDS" PKG_LONGDESC="This package contains a precompiled kernel image and the modules." From 368a9feaf870a1960930e2ec0dac46a9eb378eb1 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 23 May 2019 22:56:24 +0100 Subject: [PATCH 03/14] buildsystem: don't allow unpack folder to become visible until tagged --- scripts/clean | 9 ++++++--- scripts/unpack | 35 +++++++++++++++++++++-------------- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/scripts/clean b/scripts/clean index 9e7ca86984..9097af794d 100755 --- a/scripts/clean +++ b/scripts/clean @@ -22,10 +22,13 @@ clean_package() { build_msg "CLR_WARNING" "*" "$(print_color "CLR_WARNING_DIM" "Removing ${i} ...")" rm -rf "${i}" fi - else + elif [ -d "${i}" -a -f "${i}/.libreelec-package" ]; then # force clean if no stamp found (previous unpack failed) - build_msg "" "" "* Removing ${i} ..." - rm -rf "${i}" + . "${i}/.libreelec-package" + if [ "${INFO_PKG_NAME}" = "${1}" ]; then + build_msg "" "" "* Removing ${i} ..." + rm -rf "${i}" + fi fi done rm -f "${STAMPS}/${1}/build_"* diff --git a/scripts/unpack b/scripts/unpack index 5415acf5aa..1f4342a5f4 100755 --- a/scripts/unpack +++ b/scripts/unpack @@ -60,40 +60,46 @@ if [ -d "$SOURCES/${PKG_NAME}" -o -d "$PKG_DIR/sources" ]; then pkg_call_exists pre_unpack && pkg_call pre_unpack if pkg_call_exists unpack; then - pkg_call unpack + ( + PKG_BUILD="${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}" + pkg_call unpack + ) else if [ -n "$PKG_URL" ]; then ${SCRIPTS}/extract "${PKG_NAME}" "${PKG_UNPACK_DIR}" fi fi - _pkg_build="$(basename "${PKG_BUILD}")" - [ -z "${PKG_SOURCE_DIR}" -a -d "${PKG_UNPACK_DIR}/${_pkg_build}"* ] && mv "${PKG_UNPACK_DIR}/${_pkg_build}"* "${PKG_BUILD}" + if [ -z "${PKG_SOURCE_DIR}" -a -d "${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}"* ]; then + mv "${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}"* "${PKG_UNPACK_DIR}/.intermediate" + fi - if [ ! -d "${PKG_BUILD}" ]; then + if [ ! -d "${PKG_UNPACK_DIR}/.intermediate" ]; then if [ -n "${PKG_SOURCE_DIR}" ]; then if [ -d "${PKG_UNPACK_DIR}"/${PKG_SOURCE_DIR} ]; then - mv "${PKG_UNPACK_DIR}"/${PKG_SOURCE_DIR} "${PKG_BUILD}" + mv "${PKG_UNPACK_DIR}"/${PKG_SOURCE_DIR} "${PKG_UNPACK_DIR}/.intermediate" else # fallback - mv "${BUILD}"/${PKG_SOURCE_DIR} "${PKG_BUILD}" + mv "${BUILD}"/${PKG_SOURCE_DIR} "${PKG_UNPACK_DIR}/.intermediate" fi elif [ -d "${PKG_BUILD}"* ]; then - mv "${PKG_BUILD}"* "${PKG_BUILD}" + mv "${PKG_BUILD}"* "${PKG_UNPACK_DIR}/.intermediate" fi fi - # cleanup - rm -rf "${PKG_UNPACK_DIR}" + [ ! -d "${PKG_UNPACK_DIR}/.intermediate" ] && mkdir -p "${PKG_UNPACK_DIR}/.intermediate" if [ -d "$PKG_DIR/sources" ]; then - [ ! -d "$PKG_BUILD" ] && mkdir -p "$PKG_BUILD" - cp -PRf "$PKG_DIR/sources/"* "$PKG_BUILD" + cp -PRf "$PKG_DIR/sources/"* "${PKG_UNPACK_DIR}/.intermediate" fi - if [ -z "$PKG_URL" ]; then - mkdir -p "${PKG_BUILD}" - fi + echo "INFO_PKG_NAME=\"${PKG_NAME}\"" > "${PKG_UNPACK_DIR}/.intermediate/.libreelec-package" + + rm -fr "${PKG_BUILD}" + mv "${PKG_UNPACK_DIR}/.intermediate" "${PKG_BUILD}" + + # cleanup + rm -rf "${PKG_UNPACK_DIR}" pkg_call_exists post_unpack && pkg_call post_unpack @@ -184,6 +190,7 @@ fi if [ "$PKG_SECTION" != "virtual" ]; then mkdir -p "$PKG_BUILD" + echo "INFO_PKG_NAME=\"${PKG_NAME}\"" > "${PKG_BUILD}/.libreelec-package" rm -f $STAMPS/${PKG_NAME}/build_* From fd4f97b05ed4dc450a8d06b56e38883c407dde2b Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 23 May 2019 22:57:05 +0100 Subject: [PATCH 04/14] dotnet-runtime: redundant custom unpack() --- packages/addons/tools/dotnet-runtime/package.mk | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/addons/tools/dotnet-runtime/package.mk b/packages/addons/tools/dotnet-runtime/package.mk index b3dd996ed4..40ed87f919 100644 --- a/packages/addons/tools/dotnet-runtime/package.mk +++ b/packages/addons/tools/dotnet-runtime/package.mk @@ -36,11 +36,6 @@ PKG_ADDON_NAME=".Net Core Runtime" PKG_ADDON_TYPE="xbmc.python.script" PKG_MAINTAINER="Anton Voyl (awiouy)" -unpack() { - mkdir -p $PKG_BUILD - $SCRIPTS/extract $PKG_NAME $PKG_BUILD -} - addon() { mkdir -p $ADDON_BUILD/$PKG_ADDON_ID/bin cp -r $PKG_BUILD/$PKG_NAME-$PKG_VERSION/* \ From bf2f1bf2a12e1f8c9d65907ef9918d3dc95cc08c Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 23 May 2019 23:28:02 +0100 Subject: [PATCH 05/14] media_tree/media_tree_cc: use post_unpack() --- packages/linux-driver-addons/dvb/depends/media_tree/package.mk | 2 ++ .../linux-driver-addons/dvb/depends/media_tree_cc/package.mk | 2 ++ 2 files changed, 4 insertions(+) diff --git a/packages/linux-driver-addons/dvb/depends/media_tree/package.mk b/packages/linux-driver-addons/dvb/depends/media_tree/package.mk index 2aa920e299..ae3fd907a6 100644 --- a/packages/linux-driver-addons/dvb/depends/media_tree/package.mk +++ b/packages/linux-driver-addons/dvb/depends/media_tree/package.mk @@ -15,7 +15,9 @@ PKG_TOOLCHAIN="manual" unpack() { mkdir -p $PKG_BUILD/ tar -xf $SOURCES/$PKG_NAME/$PKG_NAME-$PKG_VERSION.tar.bz2 -C $PKG_BUILD/ +} +post_unpack() { # hack/workaround for borked upstream kernel/media_build # without removing atomisp there a lot additional includes that # slowdown build process after modpost from 3min to 6min diff --git a/packages/linux-driver-addons/dvb/depends/media_tree_cc/package.mk b/packages/linux-driver-addons/dvb/depends/media_tree_cc/package.mk index b04f4771eb..ccce3a9c7e 100644 --- a/packages/linux-driver-addons/dvb/depends/media_tree_cc/package.mk +++ b/packages/linux-driver-addons/dvb/depends/media_tree_cc/package.mk @@ -15,7 +15,9 @@ PKG_TOOLCHAIN="manual" unpack() { mkdir -p $PKG_BUILD/ tar -xf $SOURCES/$PKG_NAME/$PKG_NAME-$PKG_VERSION.tar.bz2 -C $PKG_BUILD/ +} +post_unpack() { # hack/workaround for borked upstream kernel/media_build # without removing atomisp there a lot additional includes that # slowdown build process after modpost from 3min to 6min From 7addd1c251c24b066b41e65386e745e4c25c8ac4 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 23 May 2019 23:33:17 +0100 Subject: [PATCH 06/14] firmware-imx: unpack into unique unpack folder not $BUILD --- packages/linux-firmware/firmware-imx/package.mk | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/linux-firmware/firmware-imx/package.mk b/packages/linux-firmware/firmware-imx/package.mk index 4d895b40e2..89dd714efb 100644 --- a/packages/linux-firmware/firmware-imx/package.mk +++ b/packages/linux-firmware/firmware-imx/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="firmware-imx" PKG_VERSION="5.4" @@ -13,9 +14,8 @@ PKG_LONGDESC="firmware-imx: Freescale IMX firmware such as for the VPU" PKG_TOOLCHAIN="manual" unpack() { - mkdir -p $BUILD - cd $BUILD - sh $SOURCES/$PKG_NAME/$PKG_NAME-$PKG_VERSION.bin --auto-accept + cd $(dirname $PKG_BUILD) + sh $SOURCES/$PKG_NAME/$PKG_NAME-$PKG_VERSION.bin --auto-accept } makeinstall_target() { From 7f190edc5e193b1755ebb3f47bd64b6283993689 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 24 May 2019 04:40:09 +0100 Subject: [PATCH 07/14] fontconfig: needs util-linux for uuid support --- packages/x11/other/fontconfig/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/x11/other/fontconfig/package.mk b/packages/x11/other/fontconfig/package.mk index 0dd1f233a1..6b0000b5f1 100644 --- a/packages/x11/other/fontconfig/package.mk +++ b/packages/x11/other/fontconfig/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="9f0d852b39d75fc655f9f53850eb32555394f36104a044bb2b2fc9e66dbbfa7f" PKG_LICENSE="OSS" PKG_SITE="http://www.fontconfig.org" PKG_URL="http://www.freedesktop.org/software/fontconfig/release/$PKG_NAME-$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain util-macros freetype libxml2 zlib expat" +PKG_DEPENDS_TARGET="toolchain util-linux util-macros freetype libxml2 zlib expat" PKG_LONGDESC="Fontconfig is a library for font customization and configuration." PKG_CONFIGURE_OPTS_TARGET="--with-arch=$TARGET_ARCH \ From ac1174d2b4eb94eec354f461fcf7702d8818aab6 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sat, 25 May 2019 00:35:43 +0100 Subject: [PATCH 08/14] audiodecoder.usf: don't build for RPi1, broken in #3503 --- .../mediacenter/kodi-binary-addons/audiodecoder.usf/package.mk | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/mediacenter/kodi-binary-addons/audiodecoder.usf/package.mk b/packages/mediacenter/kodi-binary-addons/audiodecoder.usf/package.mk index 3529a1507c..b28e165a36 100644 --- a/packages/mediacenter/kodi-binary-addons/audiodecoder.usf/package.mk +++ b/packages/mediacenter/kodi-binary-addons/audiodecoder.usf/package.mk @@ -17,3 +17,4 @@ PKG_LONGDESC="audiodecoder.usf" PKG_IS_ADDON="yes" PKG_ADDON_TYPE="kodi.audiodecoder" +PKG_ADDON_PROJECTS="any !RPi1" From 7e5fb76ddf207ece3ff0cebf65e248546cd6c12a Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sat, 25 May 2019 01:38:48 +0100 Subject: [PATCH 09/14] udpxy: broken since gcc-8.0 --- .../addon-depends/network-tools-depends/udpxy/package.mk | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/addons/addon-depends/network-tools-depends/udpxy/package.mk b/packages/addons/addon-depends/network-tools-depends/udpxy/package.mk index 9009849291..a8ed4de2ee 100644 --- a/packages/addons/addon-depends/network-tools-depends/udpxy/package.mk +++ b/packages/addons/addon-depends/network-tools-depends/udpxy/package.mk @@ -10,6 +10,10 @@ PKG_URL="http://www.udpxy.com/download/1_23/${PKG_NAME}.${PKG_VERSION}-prod.tar. PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A UDP-to-HTTP multicast traffic relay daemon." +configure_target() { + export CFLAGS+=" -Wno-stringop-truncation" +} + makeinstall_target() { : } From 491610dc0079b3c71539d15ec5498444d2d97e6e Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sat, 25 May 2019 12:12:26 +0100 Subject: [PATCH 10/14] screensaver.greynetic: needs glm --- .../kodi-binary-addons/screensaver.greynetic/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediacenter/kodi-binary-addons/screensaver.greynetic/package.mk b/packages/mediacenter/kodi-binary-addons/screensaver.greynetic/package.mk index 0b7090e9f7..0e443fbabe 100644 --- a/packages/mediacenter/kodi-binary-addons/screensaver.greynetic/package.mk +++ b/packages/mediacenter/kodi-binary-addons/screensaver.greynetic/package.mk @@ -10,7 +10,7 @@ PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="https://github.com/xbmc/screensaver.greynetic" PKG_URL="https://github.com/xbmc/screensaver.greynetic/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain kodi-platform" +PKG_DEPENDS_TARGET="toolchain kodi-platform glm" PKG_SECTION="" PKG_SHORTDESC="screensaver.greynetic" PKG_LONGDESC="screensaver.greynetic" From 72cdeda49ef6a8cbaac4412ccb57fdec205d3fe6 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sat, 25 May 2019 13:59:07 +0100 Subject: [PATCH 11/14] distro-tool: handle garbage in error response --- tools/distro-tool | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tools/distro-tool b/tools/distro-tool index 075a855a60..71b3b1473c 100755 --- a/tools/distro-tool +++ b/tools/distro-tool @@ -104,7 +104,13 @@ class MyUtility(object): return (0, subprocess.check_output(command.split(" "), stderr=_logfile)) except subprocess.CalledProcessError as cpe: if MyUtility.isPython3: - return (cpe.returncode, cpe.output.decode("utf-8")) + # Clean up undecodable garbage in response (eg. libftdi1 error page) + output = bytearray() + for c in cpe.output: + if c <= 127: + output.append(c) + + return (cpe.returncode, output.decode("utf-8")) else: return (cpe.returncode, cpe.output) finally: From 1d81d222253d92800a600bec5f5289536041ed51 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sun, 26 May 2019 18:50:07 +0100 Subject: [PATCH 12/14] buildsystem: add pre_configure() function that is not target specific --- packages/readme.md | 2 +- scripts/build | 1 + tools/pkgcheck | 13 +++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/readme.md b/packages/readme.md index 26bc6cce55..e67a1c6e33 100644 --- a/packages/readme.md +++ b/packages/readme.md @@ -148,7 +148,7 @@ Full list of overwrittable functions. | unpack
pre_unpack
post_unpack | - | Extract the source from the downloaded file | | pre_patch
post_patch | - | Apply the patches to the source, after extraction. The patch function it self is not allowed to overwritten | | pre_build_\[stage] | yes | Runs before of the start of the build | -| configure_\[stage]
pre_configure_\[stage]
post_configure_\[stage] | yes | Configure the package for the compile. This is only relevant for toolchain, that supports it (e.g. meson, cmake, configure, manual) | +| pre_configure
pre_configure_\[stage]
configure_\[stage]
post_configure_\[stage] | yes | Configure the package for the compile. This is only relevant for toolchain, that supports it (e.g. meson, cmake, configure, manual) | | make_\[stage]
pre_make_\[stage]
post_make_\[stage] | yes | Build of the package | | makeinstall_\[stage]
pre_makeinstall_\[stage]
post_makeinstall_\[stage] | yes | Installation of the files in the correct pathes
host: TOOLCHAIN
target: SYSROOT and IMAGE
bootstrap and init: temporary destination | addon | - | Copy all files together for addon creation. This is requiered for addons | diff --git a/scripts/build b/scripts/build index 20c804ca89..1d068051c7 100755 --- a/scripts/build +++ b/scripts/build @@ -238,6 +238,7 @@ if [ -f "$PKG_CONFIGURE_SCRIPT" -o -f "$PKG_CMAKE_SCRIPT" -o -f "$PKG_MESON_SCRI fi # configure +pkg_call_exists pre_configure && pkg_call pre_configure pkg_call_exists pre_configure_$TARGET && pkg_call pre_configure_$TARGET if pkg_call_exists configure_$TARGET; then diff --git a/tools/pkgcheck b/tools/pkgcheck index af298edc6c..55282dc9b7 100755 --- a/tools/pkgcheck +++ b/tools/pkgcheck @@ -76,8 +76,8 @@ process_line() { [ -n "${matches}" ] && log "${filename}" ${lc} "FAIL" "late binding violation" "${matches:2}" } -init_multi_func() { - local f t multi +init_target_funcs() { + local f t funcs for t in target host init bootstrap; do for f in pre_build \ @@ -85,10 +85,10 @@ init_multi_func() { pre_make make post_make \ pre_makeinstall makeinstall post_makeinstall \ ; do - multi+=" ${f}_${t}" + funcs+=" ${f}_${t}" done done - echo "${multi:1}" + echo "${funcs:1}" } check_func_name() { @@ -98,7 +98,8 @@ check_func_name() { for f in configure_package \ pre_unpack unpack post_unpack \ pre_patch post_patch \ - ${MULTI_FUNC} \ + pre_configure \ + ${TARGET_FUNCS} \ pre_install post_install \ addon \ ; do @@ -146,7 +147,7 @@ process_pkg() { done <<< "$(grep -E ".*() {" "${filename}" | sed 's/[[:space:]]*{.*//' | sort | uniq -c | grep -v ^[[:space:]]*1[[:space:]])" } -MULTI_FUNC="$(init_multi_func)" +TARGET_FUNCS="$(init_target_funcs)" if [ $# -ne 0 ]; then for arg in ${@}; do From cd0e70eb65bacfe8c3a57ebf6659c90df2f50f44 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Wed, 29 May 2019 00:49:26 +0100 Subject: [PATCH 13/14] buildsystem: replace $(cat file) with faster alternative See: https://www.gnu.org/software/bash/manual/html_node/Command-Substitution.html " ... Bash performs the expansion by executing command in a subshell environment and replacing the command substitution with the standard output of the command, with any trailing newlines deleted. Embedded newlines are not deleted, but they may be removed during word splitting. The command substitution $(cat file) can be replaced by the equivalent but faster $(< file). " Testing indicates var=$(< file) is twice as fast as var=$(cat file). --- config/functions | 6 +++--- config/multithread | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/config/functions b/config/functions index af1020e884..2c5529b70f 100644 --- a/config/functions +++ b/config/functions @@ -1264,7 +1264,7 @@ pkg_lock() { # As we now have the lock, if .failed still exists then a previous process must have failed if [ -f "${THREAD_CONTROL}/locks/${pkg}.${task}.failed" ]; then - fail_seq="$(cat "${THREAD_CONTROL}/locks/${pkg}.${task}.failed")" + fail_seq="$(< "${THREAD_CONTROL}/locks/${pkg}.${task}.failed")" print_color CLR_ERROR "FAILURE: ${pkg}.${task}.failed exists, a previous dependency process has failed (seq: ${fail_seq})\n" if [ -d "${THREAD_CONTROL}/logs" ]; then cat <>"${THREAD_CONTROL}/status"; num=$((num + 1)); done - num=$(($(cat "${THREAD_CONTROL}/progress.prev") + 1)) + num=$(< "${THREAD_CONTROL}/progress.prev") projdevarch="${PROJECT}/" [ -n "${DEVICE}" ] && projdevarch+="${DEVICE}/" projdevarch+="${TARGET_ARCH}" TZ=UTC0 printf -v elapsed "%(%H:%M:%S)T" $(($(date +%s) - MTBUILDSTART)) - printf -v preamble "%s Dashboard (%s) - %d of %d jobs completed, %s elapsed" "${DISTRONAME}" "${projdevarch}" ${num} ${MTMAXJOBS} "${elapsed}" + printf -v preamble "%s Dashboard (%s) - %d of %d jobs completed, %s elapsed" "${DISTRONAME}" "${projdevarch}" $((num + 1)) ${MTMAXJOBS} "${elapsed}" printf -v preamble "%b%-105s %s" "\e[2J\e[0;0H" "${preamble//\//\\/}" "$(date "+%Y-%m-%d %H:%M:%S")" if [ "${DISABLE_COLORS}" != "yes" ]; then diff --git a/config/multithread b/config/multithread index cb96bfa574..902c6f5a83 100644 --- a/config/multithread +++ b/config/multithread @@ -107,8 +107,9 @@ package_worker() { ( flock --exclusive 95 [ ${result} -eq 0 ] && status="DONE" || status="FAIL" - num=$(($(cat "${THREAD_CONTROL}/progress") + 1)) + num=$(< "${THREAD_CONTROL}/progress") mv "${THREAD_CONTROL}/progress" "${THREAD_CONTROL}/progress.prev" + num=$((num + 1)) echo ${num} >"${THREAD_CONTROL}/progress" printf "[%0*d/%0*d] [%-4s] %-7s %s\n" ${#jobs} ${num} ${#jobs} ${jobs} "${status}" "${task}" "${pkgname}" >&2 ) 95>"${THREAD_CONTROL}/locks/.progress" From aa7d22f05b763402e8eaad64efa6ff6876b63c20 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Wed, 29 May 2019 00:56:10 +0100 Subject: [PATCH 14/14] buildsystem: avoid process forks to init dashboard status file A typical image build will update the dashboard 3500-4500 times. This change avoids two process forks (cat, wc) per update, and the remaining $(< file) is faster than $(cat file). --- config/functions | 7 +++++-- config/multithread | 1 + 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/functions b/config/functions index 2c5529b70f..85579cf4de 100644 --- a/config/functions +++ b/config/functions @@ -1322,8 +1322,11 @@ update_dashboard() { sedline=$((MTJOBID + 2)) - num=$(cat "${THREAD_CONTROL}/status" | wc -l) - while [ ${num} -lt ${sedline} ]; do echo "" >>"${THREAD_CONTROL}/status"; num=$((num + 1)); done + num=$(< "${THREAD_CONTROL}/status.max") + if [ ${num} -lt ${sedline} ]; then + echo ${sedline} >"${THREAD_CONTROL}/status.max" + for i in $(seq $((num + 1)) ${sedline}); do echo "" >>"${THREAD_CONTROL}/status"; done + fi num=$(< "${THREAD_CONTROL}/progress.prev") projdevarch="${PROJECT}/" diff --git a/config/multithread b/config/multithread index 902c6f5a83..ca389cddbe 100644 --- a/config/multithread +++ b/config/multithread @@ -144,6 +144,7 @@ start_multithread_build() { mkdir -p "${THREAD_CONTROL}/locks" echo -1 >"${THREAD_CONTROL}/progress.prev" echo 0 >"${THREAD_CONTROL}/progress" + echo 0 >"${THREAD_CONTROL}/status.max" touch "${THREAD_CONTROL}/status" [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1