Merge pull request #3661 from MilhouseVH/le10_abs_patch_dir

buildsystem: add absolute patch dirs, remove unecessary sub-process, other cleanup

config/functions

@@ -726,7 +726,7 @@ get_pkg_variable() {
 
 # get package's build dir
 get_build_dir() {
-  local _PKG_NAME="$(get_pkg_variable "$1" PKG_NAME)" _PKG_VERSION="$(get_pkg_version "$1")"
+  local _PKG_NAME="${1%:*}" _PKG_VERSION="$(get_pkg_version "$1")"
   if [ -n "$_PKG_NAME" -a -n "$_PKG_VERSION" ]; then
     echo $BUILD/${_PKG_NAME}-${_PKG_VERSION}
   fi
@@ -1251,6 +1251,8 @@ enable_service() {
 
 
 ### MULTI-THREADED FUNCTION HELPERS ###
+# Test MTWITHLOCKS so that these functions are a no-op during non-multithreaded builds.
+
 # Prevent concurrent modifications to a package (unpack) or
 # package:target (install/build).
 #
@@ -1305,7 +1307,7 @@ pkg_lock_status() {
   (
     flock --exclusive 94
 
-    printf -v line "%s: <%05d> [%02d/%0*d] %-7s %-7s %-35s" \
+    printf -v line "%s: <%06d> [%02d/%0*d] %-7s %-7s %-35s" \
     "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}"
     [ -n "${msg}" ] && line+=" (${msg})"
 
@@ -1346,6 +1348,7 @@ update_dashboard() {
   projdevarch="${PROJECT}/"
   [ -n "${DEVICE}" ] && projdevarch+="${DEVICE}/"
   projdevarch+="${TARGET_ARCH}"
+  [ -n "${BUILD_SUFFIX}" ] && projdevarch+=", ${BUILD_SUFFIX}"
   TZ=UTC0 printf -v elapsed "%(%H:%M:%S)T" $(($(date +%s) - MTBUILDSTART))
   printf -v preamble "%s Dashboard (%s) - %d of %d jobs completed, %s elapsed" "${DISTRONAME}" "${projdevarch}" $((num + 1)) ${MTMAXJOBS} "${elapsed}"
   printf -v preamble "%b%-105s %s" "\e[2J\e[0;0H" "${preamble//\//\\/}" "$(date "+%Y-%m-%d %H:%M:%S")"
@@ -1374,7 +1377,6 @@ update_dashboard() {
 
 # Thread concurrency helpers to avoid concurrency issues with some code,
 # eg. when Python installs directly into $TOOLCHAIN.
-# Test MTJOBID so that these functions are a no-op during non-multithreaded builds.
 acquire_exclusive_lock() {
   [ "${MTWITHLOCKS}" != "yes" ] && return 0
 

config/multithread

@@ -150,7 +150,7 @@ start_multithread_build() {
   [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1
 
   # Bootstrap GNU parallel
-  $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package"
+  MTWITHLOCKS=no $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package"
 
   # if number of detected slots is 1 then don't bother using inter-process locks as this is a sequential build
   [ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && singlethread=yes || singlethread=no

packages/linux-driver-addons/dvb/crazycat/package.mk

@@ -8,7 +8,7 @@ PKG_LICENSE="GPL"
 PKG_SITE="https://bitbucket.org/CrazyCat/media_build"
 PKG_URL="https://bitbucket.org/CrazyCat/media_build/get/$PKG_VERSION.tar.gz"
 PKG_DEPENDS_TARGET="toolchain linux media_tree_cc"
-PKG_NEED_UNPACK="$LINUX_DEPENDS media_tree_cc"
+PKG_NEED_UNPACK="$LINUX_DEPENDS $(get_pkg_directory media_tree_cc)"
 PKG_SECTION="driver.dvb"
 PKG_LONGDESC="DVB driver for TBS cards with CrazyCats additions"
 

packages/linux-driver-addons/dvb/dvb-latest/package.mk

@@ -8,7 +8,7 @@ PKG_LICENSE="GPL"
 PKG_SITE="http://git.linuxtv.org/media_build.git"
 PKG_URL="https://git.linuxtv.org/media_build.git/snapshot/${PKG_VERSION}.tar.gz"
 PKG_DEPENDS_TARGET="toolchain linux media_tree"
-PKG_NEED_UNPACK="$LINUX_DEPENDS media_tree"
+PKG_NEED_UNPACK="$LINUX_DEPENDS $(get_pkg_directory media_tree)"
 PKG_SECTION="driver.dvb"
 PKG_LONGDESC="DVB drivers from the latest kernel (media_build)"
 

packages/linux/package.mk

@@ -127,6 +127,7 @@ pre_make_target() {
     rm -rf $BUILD/initramfs
     $SCRIPTS/install initramfs
   )
+  pkg_lock_status "ACTIVE" "linux:target" "build"
 
   if [ "$TARGET_ARCH" = "x86_64" ]; then
     # copy some extra firmware to linux tree

scripts/get_archive

@@ -18,6 +18,8 @@ _get_file_already_downloaded && exit 0
 # At this point, we need to download something...
 build_msg "CLR_GET" "GET" "${1} (archive)" "indent"
 
+pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "downloading package..."
+
 PACKAGE_MIRROR="${DISTRO_MIRROR}/${PKG_NAME}/${PKG_SOURCE_NAME}"
 [ "${VERBOSE}" != "yes" ] && WGET_OPT=-q
 WGET_CMD="wget --output-file=- --timeout=30 --tries=3 --passive-ftp --no-check-certificate -c ${WGET_OPT} --progress=bar:force --show-progress -O ${PACKAGE}"

scripts/get_file

@@ -2,3 +2,5 @@
 # Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv)
 
 build_msg "CLR_GET" "GET" "${1} (file)" "indent"
+
+pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "processing package file..."

scripts/get_git

@@ -44,6 +44,8 @@ _get_repo_already_downloaded && exit 0
 # At this point, we need to download something...
 build_msg "CLR_GET" "GET" "${1} (git)" "indent"
 
+pkg_lock_status "GETPKG" "${PKG_NAME}" "unpack" "processing package repository..."
+
 rm -f "${STAMP_URL}" "${STAMP_SHA}"
 
 GIT_CLONE_PARAMS=""

scripts/unpack

@@ -17,6 +17,8 @@ PARENT_PKG="${2:-${PKG_NAME}}"
 
 pkg_lock "${PKG_NAME}" "unpack" "${PARENT_PKG}"
 
+${SCRIPTS}/get "${PKG_NAME}"
+
 if [ -n "${PKG_DEPENDS_UNPACK}" ]; then
   for p in ${PKG_DEPENDS_UNPACK}; do
     ${SCRIPTS}/unpack "${p}" "${PARENT_PKG}"
@@ -53,9 +55,7 @@ fi
 
 pkg_lock_status "ACTIVE" "${PKG_NAME}" "unpack"
 
-${SCRIPTS}/get "${PKG_NAME}"
-
-if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
+if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ] || pkg_call_exists unpack; then
   build_msg "CLR_UNPACK" "UNPACK" "${PKG_NAME}" "indent"
 
   # unpack into a unique location as unpacking into a single ${BUILD} directory is not thread-safe
@@ -63,13 +63,15 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
   rm -rf "${PKG_UNPACK_DIR}"
   mkdir -p "${PKG_UNPACK_DIR}"
 
+  # Save PKG_BUILD and point at our private unpack directory so that any
+  # modifications to the content of ${PKG_BUILD} will be "safe".
+  PKG_BUILD_ORIG="${PKG_BUILD}"
+  PKG_BUILD="${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}"
+
   pkg_call_exists pre_unpack && pkg_call pre_unpack
 
   if pkg_call_exists unpack; then
-    (
-      PKG_BUILD="${PKG_UNPACK_DIR}/${PKG_NAME}-${PKG_VERSION}"
-      pkg_call unpack
-    )
+    pkg_call unpack
   else
     if [ -n "${PKG_URL}" ]; then
       ${SCRIPTS}/extract "${PKG_NAME}" "${PKG_UNPACK_DIR}"
@@ -88,8 +90,6 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
         # fallback
         mv "${BUILD}"/${PKG_SOURCE_DIR} "${PKG_UNPACK_DIR}/.intermediate"
       fi
-    elif [ -d "${PKG_BUILD}"* ]; then
-      mv "${PKG_BUILD}"* "${PKG_UNPACK_DIR}/.intermediate"
     fi
   fi
 
@@ -99,8 +99,11 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
     cp -PRf "${PKG_DIR}/sources/"* "${PKG_UNPACK_DIR}/.intermediate"
   fi
 
+  # Add a tag to the unpacked folder before transferring into the shared build folder
   echo "INFO_PKG_NAME=\"${PKG_NAME}\"" > "${PKG_UNPACK_DIR}/.intermediate/.libreelec-package"
 
+  # Restore original PKG_BUILD, and transfer the unpacked folder
+  PKG_BUILD="${PKG_BUILD_ORIG}"
   rm -fr "${PKG_BUILD}"
   mv "${PKG_UNPACK_DIR}/.intermediate" "${PKG_BUILD}"
 
@@ -122,9 +125,14 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
     PATCH_DIRS_PRJ=""
     if [ -n "${PKG_PATCH_DIRS}" ]; then
       for patch_dir in ${PKG_PATCH_DIRS}; do
-        [ -d ${PKG_DIR}/patches/${patch_dir} ] && PATCH_DIRS_PKG+=" ${PKG_DIR}/patches/${patch_dir}/*.patch"
-        [ -d ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir}/*.patch"
-        [ -d ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir}/*.patch"
+        if [[ ${patch_dir} =~ ^/ ]]; then
+          [ -f ${patch_dir} ] && PATCH_DIRS_PKG+=" ${patch_dir}"
+          [ -d ${patch_dir} ] && PATCH_DIRS_PKG+=" ${patch_dir}/*.patch"
+        else
+          [ -d ${PKG_DIR}/patches/${patch_dir} ] && PATCH_DIRS_PKG+=" ${PKG_DIR}/patches/${patch_dir}/*.patch"
+          [ -d ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/patches/${PKG_NAME}/${patch_dir}/*.patch"
+          [ -d ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir} ] && PATCH_DIRS_PRJ+=" ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/patches/${PKG_NAME}/${patch_dir}/*.patch"
+        fi
       done
     fi
 
@@ -165,7 +173,7 @@ if [ -d "${SOURCES}/${PKG_NAME}" -o -d "${PKG_DIR}/sources" ]; then
         elif [[ "${thisdir}" =~ ^${PROJECT_DIR}/.* ]]; then
           PATCH_DESC="(project - $(basename "${thisdir}"))"
         else
-          PATCH_DESC="(unknown - $(basename "${thisdir}"))"
+          PATCH_DESC="(absolute - ${i})"
         fi
       fi
 

tools/mtstats.py

@@ -63,7 +63,7 @@ for event in events:
                                   "start": 0.0,
                                   "total": 0.0})
 
-    if event.status in ["ACTIVE", "MUTEX"]:
+    if event.status in ["ACTIVE", "MUTEX", "GETPKG"]:
         if slot["active"] == False:
             active += 1
             concurrent = concurrency.get(active, {"start": 0.0, "total": 0.0})
@@ -98,7 +98,7 @@ print("Total Build Time: %s\n" % secs_to_hms(elapsed, blankzero=False))
 
 print("Peak concurrency: %d out of %d slots\n" % (peak, len(slots)))
 
-print("Slot usage (time in ACTIVE state):       | Concurrency breakdown:\n")
+print("Slot usage (time in an \"active\" state):  | Concurrency breakdown:\n")
 print("#Rank  Slot  Usage        ( Pct )          # of Slots  Usage        ( Pct )")
 
 lines = []