diff --git a/config/functions b/config/functions index f11f4ce60d..419e3c5358 100644 --- a/config/functions +++ b/config/functions @@ -726,7 +726,7 @@ get_pkg_variable() { # get package's build dir get_build_dir() { - local _PKG_NAME="$(get_pkg_variable "$1" PKG_NAME)" _PKG_VERSION="$(get_pkg_version "$1")" + local _PKG_NAME="${1%:*}" _PKG_VERSION="$(get_pkg_version "$1")" if [ -n "$_PKG_NAME" -a -n "$_PKG_VERSION" ]; then echo $BUILD/${_PKG_NAME}-${_PKG_VERSION} fi @@ -1251,6 +1251,8 @@ enable_service() { ### MULTI-THREADED FUNCTION HELPERS ### +# Test MTWITHLOCKS so that these functions are a no-op during non-multithreaded builds. + # Prevent concurrent modifications to a package (unpack) or # package:target (install/build). # @@ -1305,7 +1307,7 @@ pkg_lock_status() { ( flock --exclusive 94 - printf -v line "%s: <%05d> [%02d/%0*d] %-7s %-7s %-35s" \ + printf -v line "%s: <%06d> [%02d/%0*d] %-7s %-7s %-35s" \ "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" [ -n "${msg}" ] && line+=" (${msg})" @@ -1346,6 +1348,7 @@ update_dashboard() { projdevarch="${PROJECT}/" [ -n "${DEVICE}" ] && projdevarch+="${DEVICE}/" projdevarch+="${TARGET_ARCH}" + [ -n "${BUILD_SUFFIX}" ] && projdevarch+=", ${BUILD_SUFFIX}" TZ=UTC0 printf -v elapsed "%(%H:%M:%S)T" $(($(date +%s) - MTBUILDSTART)) printf -v preamble "%s Dashboard (%s) - %d of %d jobs completed, %s elapsed" "${DISTRONAME}" "${projdevarch}" $((num + 1)) ${MTMAXJOBS} "${elapsed}" printf -v preamble "%b%-105s %s" "\e[2J\e[0;0H" "${preamble//\//\\/}" "$(date "+%Y-%m-%d %H:%M:%S")" @@ -1374,7 +1377,6 @@ update_dashboard() { # Thread concurrency helpers to avoid concurrency issues with some code, # eg. when Python installs directly into $TOOLCHAIN. -# Test MTJOBID so that these functions are a no-op during non-multithreaded builds. acquire_exclusive_lock() { [ "${MTWITHLOCKS}" != "yes" ] && return 0 diff --git a/config/multithread b/config/multithread index ca389cddbe..18b24f959d 100644 --- a/config/multithread +++ b/config/multithread @@ -150,7 +150,7 @@ start_multithread_build() { [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1 # Bootstrap GNU parallel - $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package" + MTWITHLOCKS=no $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package" # if number of detected slots is 1 then don't bother using inter-process locks as this is a sequential build [ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && singlethread=yes || singlethread=no diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXi/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXi/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXrender/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXrender/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXtst/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libXtst/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libxcb/sources/trigger_unpack b/packages/addons/addon-depends/chrome-depends/chrome-libxcb/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/addons/addon-depends/dotnet-runtime-depends/curl3/sources/trigger_unpack b/packages/addons/addon-depends/dotnet-runtime-depends/curl3/sources/trigger_unpack deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/packages/linux-driver-addons/dvb/crazycat/package.mk b/packages/linux-driver-addons/dvb/crazycat/package.mk index 8889791b6e..ee9ec8253d 100644 --- a/packages/linux-driver-addons/dvb/crazycat/package.mk +++ b/packages/linux-driver-addons/dvb/crazycat/package.mk @@ -8,7 +8,7 @@ PKG_LICENSE="GPL" PKG_SITE="https://bitbucket.org/CrazyCat/media_build" PKG_URL="https://bitbucket.org/CrazyCat/media_build/get/$PKG_VERSION.tar.gz" PKG_DEPENDS_TARGET="toolchain linux media_tree_cc" -PKG_NEED_UNPACK="$LINUX_DEPENDS media_tree_cc" +PKG_NEED_UNPACK="$LINUX_DEPENDS $(get_pkg_directory media_tree_cc)" PKG_SECTION="driver.dvb" PKG_LONGDESC="DVB driver for TBS cards with CrazyCats additions" diff --git a/packages/linux-driver-addons/dvb/dvb-latest/package.mk b/packages/linux-driver-addons/dvb/dvb-latest/package.mk index d8b8b8997f..9685def240 100644 --- a/packages/linux-driver-addons/dvb/dvb-latest/package.mk +++ b/packages/linux-driver-addons/dvb/dvb-latest/package.mk @@ -8,7 +8,7 @@ PKG_LICENSE="GPL" PKG_SITE="http://git.linuxtv.org/media_build.git" PKG_URL="https://git.linuxtv.org/media_build.git/snapshot/${PKG_VERSION}.tar.gz" PKG_DEPENDS_TARGET="toolchain linux media_tree" -PKG_NEED_UNPACK="$LINUX_DEPENDS media_tree" +PKG_NEED_UNPACK="$LINUX_DEPENDS $(get_pkg_directory media_tree)" PKG_SECTION="driver.dvb" PKG_LONGDESC="DVB drivers from the latest kernel (media_build)" diff --git a/packages/linux/package.mk b/packages/linux/package.mk index 2e75445872..87982a2c6b 100644 --- a/packages/linux/package.mk +++ b/packages/linux/package.mk @@ -127,6 +127,7 @@ pre_make_target() { rm -rf $BUILD/initramfs $SCRIPTS/install initramfs ) + pkg_lock_status "ACTIVE" "linux:target" "build" if [ "$TARGET_ARCH" = "x86_64" ]; then # copy some extra firmware to linux tree diff --git a/scripts/get_archive b/scripts/get_archive index eb912015a5..85ec6c545f 100755 --- a/scripts/get_archive +++ b/scripts/get_archive @@ -18,6 +18,8 @@ _get_file_already_downloaded && exit 0 # At this point, we need to download something... build_msg "CLR_GET" "GET" "${1} (archive)" "indent" +pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "downloading package..." + PACKAGE_MIRROR="${DISTRO_MIRROR}/${PKG_NAME}/${PKG_SOURCE_NAME}" [ "${VERBOSE}" != "yes" ] && WGET_OPT=-q WGET_CMD="wget --output-file=- --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c ${WGET_OPT} --progress=bar:force --show-progress -O ${PACKAGE}" diff --git a/scripts/get_file b/scripts/get_file index 2a34dd545b..02c0021df4 100755 --- a/scripts/get_file +++ b/scripts/get_file @@ -2,3 +2,5 @@ # Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv) build_msg "CLR_GET" "GET" "${1} (file)" "indent" + +pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "processing package file..." diff --git a/scripts/get_git b/scripts/get_git index 68bf92023a..dd8486bf34 100755 --- a/scripts/get_git +++ b/scripts/get_git @@ -44,6 +44,8 @@ _get_repo_already_downloaded && exit 0 # At this point, we need to download something... build_msg "CLR_GET" "GET" "${1} (git)" "indent" +pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "processing package repository..." + rm -f "${STAMP_URL}" "${STAMP_SHA}" GIT_CLONE_PARAMS="" diff --git a/scripts/unpack b/scripts/unpack index d3aa480329..c88bf0769d 100755 --- a/scripts/unpack +++ b/scripts/unpack @@ -17,6 +17,8 @@ PARENT_PKG="${2:-${PKG_NAME}}" pkg_lock "${PKG_NAME}" "unpack" "${PARENT_PKG}" +${SCRIPTS}/get "${PKG_NAME}" + if [ -n "${PKG_DEPENDS_UNPACK}" ]; then for p in ${PKG_DEPENDS_UNPACK}; do ${SCRIPTS}/unpack "${p}" "${PARENT_PKG}" @@ -53,9 +55,7 @@ fi pkg_lock_status "ACTIVE" "${PKG_NAME}" "unpack" -${SCRIPTS}/get "${PKG_NAME}" - -if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then +if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ] || pkg_call_exists unpack; then build_msg "CLR_UNPACK" "UNPACK" "${PKG_NAME}" "indent" # unpack into a unique location as unpacking into a single ${BUILD} directory is not thread-safe @@ -63,13 +63,15 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then rm -rf "${PKG_UNPACK_DIR}" mkdir -p "${PKG_UNPACK_DIR}" + # Save PKG_BUILD and point at our private unpack directory so that any + # modifications to the content of ${PKG_BUILD} will be "safe". + PKG_BUILD_ORIG="${PKG_BUILD}" + PKG_BUILD="${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}" + pkg_call_exists pre_unpack && pkg_call pre_unpack if pkg_call_exists unpack; then - ( - PKG_BUILD="${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}" - pkg_call unpack - ) + pkg_call unpack else if [ -n "${PKG_URL}" ]; then ${SCRIPTS}/extract "${PKG_NAME}" "${PKG_UNPACK_DIR}" @@ -88,8 +90,6 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then # fallback mv "${BUILD}"/${PKG_SOURCE_DIR} "${PKG_UNPACK_DIR}/.intermediate" fi - elif [ -d "${PKG_BUILD}"* ]; then - mv "${PKG_BUILD}"* "${PKG_UNPACK_DIR}/.intermediate" fi fi @@ -99,8 +99,11 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then cp -PRf "${PKG_DIR}/sources/"* "${PKG_UNPACK_DIR}/.intermediate" fi + # Add a tag to the unpacked folder before transferring into the shared build folder echo "INFO_PKG_NAME=\"${PKG_NAME}\"" > "${PKG_UNPACK_DIR}/.intermediate/.libreelec-package" + # Restore original PKG_BUILD, and transfer the unpacked folder + PKG_BUILD="${PKG_BUILD_ORIG}" rm -fr "${PKG_BUILD}" mv "${PKG_UNPACK_DIR}/.intermediate" "${PKG_BUILD}" @@ -122,9 +125,14 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then PATCH_DIRS_PRJ="" if [ -n "${PKG_PATCH_DIRS}" ]; then for patch_dir in ${PKG_PATCH_DIRS}; do - [ -d ${PKG_DIR}/patches/${patch_dir} ] && PATCH_DIRS_PKG+=" ${PKG_DIR}/patches/${patch_dir}/*.patch" - [ -d ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir}/*.patch" - [ -d ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir}/*.patch" + if [[ ${patch_dir} =~ ^/ ]]; then + [ -f ${patch_dir} ] && PATCH_DIRS_PKG+=" ${patch_dir}" + [ -d ${patch_dir} ] && PATCH_DIRS_PKG+=" ${patch_dir}/*.patch" + else + [ -d ${PKG_DIR}/patches/${patch_dir} ] && PATCH_DIRS_PKG+=" ${PKG_DIR}/patches/${patch_dir}/*.patch" + [ -d ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir}/*.patch" + [ -d ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir}/*.patch" + fi done fi @@ -165,7 +173,7 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then elif [[ "${thisdir}" =~ ^${PROJECT_DIR}/.* ]]; then PATCH_DESC="(project - $(basename "${thisdir}"))" else - PATCH_DESC="(unknown - $(basename "${thisdir}"))" + PATCH_DESC="(absolute - ${i})" fi fi diff --git a/tools/mtstats.py b/tools/mtstats.py index f9fd9d47c4..8fdc4ff9b2 100755 --- a/tools/mtstats.py +++ b/tools/mtstats.py @@ -63,7 +63,7 @@ for event in events: "start": 0.0, "total": 0.0}) - if event.status in ["ACTIVE", "MUTEX"]: + if event.status in ["ACTIVE", "MUTEX", "GETPKG"]: if slot["active"] == False: active += 1 concurrent = concurrency.get(active, {"start": 0.0, "total": 0.0}) @@ -98,7 +98,7 @@ print("Total Build Time: %s\n" % secs_to_hms(elapsed, blankzero=False)) print("Peak concurrency: %d out of %d slots\n" % (peak, len(slots))) -print("Slot usage (time in ACTIVE state): | Concurrency breakdown:\n") +print("Slot usage (time in an \"active\" state): | Concurrency breakdown:\n") print("#Rank Slot Usage ( Pct ) # of Slots Usage ( Pct )") lines = []