diff --git a/config/functions b/config/functions index 5597f3ddde..a1c08e890d 100644 --- a/config/functions +++ b/config/functions @@ -254,12 +254,12 @@ setup_toolchain() { fi # parallel - if ! flag_enabled "parallel" "yes"; then - NINJA_OPTS="$NINJA_OPTS -j1" - export MAKEFLAGS="-j1" - else + if flag_enabled "parallel" "yes"; then NINJA_OPTS="$NINJA_OPTS -j$CONCURRENCY_MAKE_LEVEL" export MAKEFLAGS="-j$CONCURRENCY_MAKE_LEVEL" + else + NINJA_OPTS="$NINJA_OPTS -j1" + export MAKEFLAGS="-j1" fi # verbose flag @@ -1264,7 +1264,7 @@ pkg_lock() { local pkg="$1" task="$2" parent_pkg="$3" local this_job="${MTJOBID}" - local lock_job lock_seq lock_task lock_pkg locked=no + local lock_job lock_seq lock_task lock_pkg locked=no idwidth local fail_seq exec 98>"${THREAD_CONTROL}/locks/${pkg}.${task}" @@ -1275,7 +1275,8 @@ pkg_lock() { done if [ "${locked}" = "no" -a "${lock_job}/${lock_seq}" != "${this_job}/${PARALLEL_SEQ}" ]; then - pkg_lock_status "STALLED" "${parent_pkg}" "${task}" "$(printf "waiting on [%02d] %s %s" ${lock_job} "${lock_task}" "${lock_pkg}")" + [ "${THREADCOUNT}" = "0" ] && idwidth=${#MTMAXJOBS} || idwidth=2 + pkg_lock_status "STALLED" "${parent_pkg}" "${task}" "$(printf "waiting on [%0*d] %s %s" ${idwidth} ${lock_job} "${lock_task}" "${lock_pkg}")" flock --exclusive 98 fi @@ -1303,13 +1304,15 @@ pkg_lock_status() { [ "${MTWITHLOCKS}" != "yes" ] && return 0 local status="$1" pkg="$2" task="$3" msg="$4" - local this_job="${MTJOBID}" line + local this_job="${MTJOBID}" line idwidth + + [ "${THREADCOUNT}" = "0" ] && idwidth=${#MTMAXJOBS} || idwidth=2 ( flock --exclusive 94 - printf -v line "%s: <%06d> [%02d/%0*d] %-7s %-7s %-35s" \ - "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" + printf -v line "%s: <%06d> [%0*d/%0*d] %-7s %-7s %-35s" \ + "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${idwidth} ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" [ -n "${msg}" ] && line+=" (${msg})" echo "${line}" >>"${THREAD_CONTROL}/history" @@ -1335,10 +1338,12 @@ update_dashboard() { local status="$1" pkg="$2" task="$3" msg="$4" local line sedline preamble num elapsed projdevarch - local boldred boldgreen boldyellow endcolor + local boldred boldgreen boldyellow endcolor idwidth sedline=$((MTJOBID + 2)) + [ "${THREADCOUNT}" = "0" ] && idwidth=${#MTMAXJOBS} || idwidth=2 + num=$(< "${THREAD_CONTROL}/status.max") if [ ${num} -lt ${sedline} ]; then echo ${sedline} >"${THREAD_CONTROL}/status.max" @@ -1370,7 +1375,7 @@ update_dashboard() { esac fi - printf -v line "[%02d\/%0*d] %b%-7s%b %-7s %-35s" ${MTJOBID} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${color}" "${status//\//\\/}" "${endcolor}" "${task}" "${pkg}" + printf -v line "[%0*d\/%0*d] %b%-7s%b %-7s %-35s" ${idwidth} ${MTJOBID} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${color}" "${status//\//\\/}" "${endcolor}" "${task}" "${pkg}" [ -n "${msg}" ] && line+=" ${msg//\//\\/}" sed -e "1s/.*/${preamble}/;${sedline}s/.*/${line}/" -i "${THREAD_CONTROL}/status" @@ -1383,7 +1388,7 @@ acquire_exclusive_lock() { local pkg="$1" task="$2" lockfile="${3:-global}" local this_job="${MTJOBID}" - local lock_job lock_seq lock_task lock_pkg locked=no + local lock_job lock_seq lock_task lock_pkg locked=no idwidth exec 96>"${THREAD_CONTROL}/locks/.mutex.${lockfile}" while [ : ]; do @@ -1393,7 +1398,8 @@ acquire_exclusive_lock() { done if [ "${locked}" = "no" -a "${lock_job}/${lock_seq}" != "${this_job}/${PARALLEL_SEQ}" ]; then - pkg_lock_status "MUTEX/W" "${pkg}" "${task}" "$(printf "mutex: %s; waiting on [%02d] %s %s" "${lockfile}" ${lock_job} "${lock_task}" "${lock_pkg}")" + [ "${THREADCOUNT}" = "0" ] && idwidth=${#MTMAXJOBS} || idwidth=2 + pkg_lock_status "MUTEX/W" "${pkg}" "${task}" "$(printf "mutex: %s; waiting on [%0*d] %s %s" "${lockfile}" ${idwidth} ${lock_job} "${lock_task}" "${lock_pkg}")" flock --exclusive 96 fi diff --git a/config/multithread b/config/multithread index 18b24f959d..ca30caf7a6 100644 --- a/config/multithread +++ b/config/multithread @@ -147,15 +147,16 @@ start_multithread_build() { echo 0 >"${THREAD_CONTROL}/status.max" touch "${THREAD_CONTROL}/status" - [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1 + # Increase file descriptors if building one thread/package + [ "${THREADCOUNT}" = "0" ] && ulimit -n ${ULIMITN:-10240} # Bootstrap GNU parallel MTWITHLOCKS=no $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package" - # if number of detected slots is 1 then don't bother using inter-process locks as this is a sequential build + # determine number of available slots for the given THREADCOUNT - optimise logging for single threaded builds [ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && singlethread=yes || singlethread=no - # create a single log file by default if not using locks (single build process), or the builder is a masochist + # create a single log file by default for a single threaded build (or the builder is a masochist) if [ "${singlethread}" = "yes" -a "${ONELOG,,}" != "no" ] || [ "${ONELOG,,}" = "yes" ]; then buildopts+=" --ungroup" else @@ -174,6 +175,8 @@ start_multithread_build() { ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || result=1 if [ ${result} -eq 0 ]; then + save_build_config + cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \ MTBUILDSTART=$(date +%s) MTWITHLOCKS=yes ${TOOLCHAIN}/bin/parallel \ --plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" --plus ${buildopts} \ diff --git a/packages/addons/addon-depends/chrome-depends/icu/package.mk b/packages/addons/addon-depends/chrome-depends/icu/package.mk index 12bdef5227..a356b4ad01 100644 --- a/packages/addons/addon-depends/chrome-depends/icu/package.mk +++ b/packages/addons/addon-depends/chrome-depends/icu/package.mk @@ -7,6 +7,7 @@ PKG_SHA256="d007f89ae8a2543a53525c74359b65b36412fa84b3349f1400be6dcf409fafef" PKG_LICENSE="Custom" PKG_SITE="http://www.icu-project.org" PKG_URL="http://download.icu-project.org/files/icu4c/${PKG_VERSION}/icu4c-${PKG_VERSION//./_}-src.tgz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain icu:host" PKG_LONGDESC="International Components for Unicode library." diff --git a/packages/addons/addon-depends/chrome-depends/shared-mime-info/package.mk b/packages/addons/addon-depends/chrome-depends/shared-mime-info/package.mk index 427b3da243..1ce6801959 100644 --- a/packages/addons/addon-depends/chrome-depends/shared-mime-info/package.mk +++ b/packages/addons/addon-depends/chrome-depends/shared-mime-info/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="5c0133ec4e228e41bdf52f726d271a2d821499c2ab97afd3aa3d6cf43efcdc83" PKG_LICENSE="GPL2" PKG_SITE="https://freedesktop.org/wiki/Software/shared-mime-info/" PKG_URL="http://freedesktop.org/~hadess/shared-mime-info-$PKG_VERSION.tar.xz" -PKG_DEPENDS_TARGET="toolchain libxml2" +PKG_DEPENDS_TARGET="toolchain glib libxml2" PKG_LONGDESC="The shared-mime-info package contains the core database of common types." PKG_BUILD_FLAGS="-parallel" diff --git a/packages/addons/addon-depends/libzip/package.mk b/packages/addons/addon-depends/libzip/package.mk index 026805e4a1..587fbaacd0 100644 --- a/packages/addons/addon-depends/libzip/package.mk +++ b/packages/addons/addon-depends/libzip/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="libzip" PKG_VERSION="0.11.2" @@ -7,7 +8,7 @@ PKG_SHA256="7cfbbc2c540e154b933b6e9ec781e2671086bd8114eb744ae1a1ade34d2bb6bb" PKG_LICENSE="GPL" PKG_SITE="http://www.nih.at/libzip/" PKG_URL="http://www.nih.at/libzip/${PKG_NAME}-${PKG_VERSION}.tar.xz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain zlib" PKG_LONGDESC="A C library for reading, creating, and modifying zip archives." PKG_TOOLCHAIN="configure" diff --git a/packages/addons/addon-depends/multimedia-tools-depends/opencaster/package.mk b/packages/addons/addon-depends/multimedia-tools-depends/opencaster/package.mk index 67ad14ecf0..5deff8ae96 100644 --- a/packages/addons/addon-depends/multimedia-tools-depends/opencaster/package.mk +++ b/packages/addons/addon-depends/multimedia-tools-depends/opencaster/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="c9d77f98b31d53f521e3179003a9cb66b0586704717e9d401f3bc0dafa243865" PKG_LICENSE="GPL" PKG_SITE="http://www.avalpa.com/the-key-values/15-free-software/33-opencaster" PKG_URL="http://ftp.de.debian.org/debian/pool/main/o/opencaster/opencaster_${PKG_VERSION}+dfsg.orig.tar.gz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain zlib" PKG_LONGDESC="A free and open source MPEG2 transport stream data generator and packet manipulator." pre_configure_target() { diff --git a/packages/addons/addon-depends/multimedia-tools-depends/tsdecrypt/package.mk b/packages/addons/addon-depends/multimedia-tools-depends/tsdecrypt/package.mk index 842d8fca9e..9181ebafa5 100644 --- a/packages/addons/addon-depends/multimedia-tools-depends/tsdecrypt/package.mk +++ b/packages/addons/addon-depends/multimedia-tools-depends/tsdecrypt/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2012 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="tsdecrypt" PKG_VERSION="10.0" @@ -7,7 +8,7 @@ PKG_SHA256="a337a7d60cc8f78b9dffbd7d675390497763bcb8f878ec9f1bec3eb80f32b1f1" PKG_LICENSE="GPL" PKG_SITE="http://georgi.unixsol.org/programs/tsdecrypt" PKG_URL="http://georgi.unixsol.org/programs/tsdecrypt/${PKG_NAME}-${PKG_VERSION}.tar.bz2" -PKG_DEPENDS_TARGET="toolchain libdvbcsa" +PKG_DEPENDS_TARGET="toolchain libdvbcsa openssl" PKG_LONGDESC="A tool that reads incoming mpeg transport stream over UDP/RTP and then decrypts it using libdvbcsa/ffdecsa." make_target() { diff --git a/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk b/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk index c4fcbff46c..7bc44bd2a6 100644 --- a/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk +++ b/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="dd779b6992de37995555e1d54caf0716a694765efc65480eed2c713105ab46fe" PKG_LICENSE="GPL" PKG_SITE="http://www.liblognorm.com" PKG_URL="https://github.com/rsyslog/liblognorm/archive/v$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain libestr" +PKG_DEPENDS_TARGET="toolchain libestr libfastjson" PKG_TOOLCHAIN="autotools" PKG_LONGDESC="A fast samples-based log normalization library." diff --git a/packages/addons/addon-depends/system-tools-depends/depends/efivar/package.mk b/packages/addons/addon-depends/system-tools-depends/depends/efivar/package.mk index e98a69a833..9dfc879c6f 100644 --- a/packages/addons/addon-depends/system-tools-depends/depends/efivar/package.mk +++ b/packages/addons/addon-depends/system-tools-depends/depends/efivar/package.mk @@ -8,6 +8,7 @@ PKG_ARCH="x86_64" PKG_LICENSE="LGPL" PKG_SITE="https://github.com/rhboot/efivar" PKG_URL="https://github.com/rhboot/efivar/archive/$PKG_VERSION.tar.gz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain efivar:host" PKG_LONGDESC="Tools and library to manipulate EFI variables." diff --git a/packages/addons/addon-depends/system-tools-depends/file/package.mk b/packages/addons/addon-depends/system-tools-depends/file/package.mk index d94e2ce301..67c67276d7 100644 --- a/packages/addons/addon-depends/system-tools-depends/file/package.mk +++ b/packages/addons/addon-depends/system-tools-depends/file/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="acc16f878576ad0bb29bbb7c724e29d2827f14ddb39fe94d76c3a859d4a3d0d9" PKG_LICENSE="BSD" PKG_SITE="http://www.darwinsys.com/file/" PKG_URL="https://github.com/file/file/archive/${PKG_VERSION}.tar.gz" -PKG_DEPENDS_HOST="ccache:host" +PKG_DEPENDS_HOST="toolchain" PKG_DEPENDS_TARGET="toolchain file:host zlib" PKG_LONGDESC="The file utility is used to determine the types of various files." PKG_TOOLCHAIN="autotools" diff --git a/packages/devel/glib/package.mk b/packages/devel/glib/package.mk index d557cb1b31..26043f9edd 100644 --- a/packages/devel/glib/package.mk +++ b/packages/devel/glib/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="f8d827955f0d8e197ff5c2105dd6ac4f6b63d15cd021eb1de66534c92a762161" PKG_LICENSE="LGPL" PKG_SITE="http://www.gtk.org/" PKG_URL="http://ftp.gnome.org/pub/gnome/sources/glib/${PKG_VERSION%.*}/$PKG_NAME-$PKG_VERSION.tar.xz" -PKG_DEPENDS_HOST="libffi:host Python3:host" +PKG_DEPENDS_HOST="libffi:host Python3:host meson:host" PKG_DEPENDS_TARGET="toolchain pcre zlib libffi Python3:host util-linux" PKG_LONGDESC="A library which includes support routines for C such as lists, trees, hashes, memory allocation." PKG_TOOLCHAIN="meson" diff --git a/packages/emulation/libretro-fsuae/package.mk b/packages/emulation/libretro-fsuae/package.mk index 484b25368b..861b18412f 100644 --- a/packages/emulation/libretro-fsuae/package.mk +++ b/packages/emulation/libretro-fsuae/package.mk @@ -8,7 +8,7 @@ PKG_ARCH="x86_64" PKG_LICENSE="GPLv3" PKG_SITE="https://github.com/libretro/libretro-fsuae" PKG_URL="https://github.com/libretro/libretro-fsuae/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain libmpeg2 openal-soft" +PKG_DEPENDS_TARGET="toolchain glib libmpeg2 openal-soft" PKG_LONGDESC="FS-UAE amiga emulator." PKG_BUILD_FLAGS="-lto" PKG_TOOLCHAIN="autotools" diff --git a/packages/emulation/libretro-uae4arm/package.mk b/packages/emulation/libretro-uae4arm/package.mk index 6666c36f05..28394b7e6a 100644 --- a/packages/emulation/libretro-uae4arm/package.mk +++ b/packages/emulation/libretro-uae4arm/package.mk @@ -8,7 +8,7 @@ PKG_ARCH="arm" PKG_LICENSE="GPL" PKG_SITE="https://github.com/libretro/uae4arm-libretro" PKG_URL="https://github.com/libretro/uae4arm-libretro/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain zlib" PKG_LONGDESC="UAE4ARM amiga emulator." PKG_LIBNAME="uae4arm_libretro.so" diff --git a/packages/graphics/glew/package.mk b/packages/graphics/glew/package.mk index 31d5a68a03..d40b937958 100644 --- a/packages/graphics/glew/package.mk +++ b/packages/graphics/glew/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="04de91e7e6763039bc11940095cd9c7f880baba82196a7765f727ac05a993c95" PKG_LICENSE="BSD" PKG_SITE="http://glew.sourceforge.net/" PKG_URL="$SOURCEFORGE_SRC/glew/glew/$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tgz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain libX11" PKG_LONGDESC="A cross-platform C/C++ extension loading library." make_target() { diff --git a/scripts/get_archive b/scripts/get_archive index 85ec6c545f..58f85dd006 100755 --- a/scripts/get_archive +++ b/scripts/get_archive @@ -45,7 +45,7 @@ while [ ${NBWGET} -gt 0 ]; do done if [ ${NBWGET} -eq 0 ]; then - die "\nCant't get ${1} sources : ${PKG_URL}\nTry later!" + die "\nCannot get ${1} sources : ${PKG_URL}\nTry later!" else build_msg "CLR_INFO" "INFO" "Calculated checksum: ${CALC_SHA256}" echo "${PKG_URL}" > "${STAMP_URL}" diff --git a/scripts/image b/scripts/image index d80d042ed9..f2a8740100 100755 --- a/scripts/image +++ b/scripts/image @@ -20,7 +20,6 @@ if [ "${BOOTLOADER}" = "u-boot" -a -n "${DEVICE}" ]; then fi show_config -save_build_config ${SCRIPTS}/checkdeps diff --git a/tools/dashboard b/tools/dashboard index 825b00d9ed..b4af0df6e7 100755 --- a/tools/dashboard +++ b/tools/dashboard @@ -36,7 +36,11 @@ while [ : ]; do sleep 1.0 done - tail -Fn+0 --pid=${pid} "${tcdir}/status" 2>/dev/null + if [ "${THREADCOUNT}" = "0" ]; then + tail -Fn+0 --pid=${pid} "${tcdir}/status" 2>/dev/null | grep -vE "STALLED|IDLE" + else + tail -Fn+0 --pid=${pid} "${tcdir}/status" 2>/dev/null + fi echo done