From 7c327b2d56642649117744d34eab328285fd166b Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 28 Feb 2019 06:19:13 +0000 Subject: [PATCH 1/7] scripts/build: fix broken sym links due to isolated sysroot --- scripts/build | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/build b/scripts/build index a4be68d83f..f96a07f7f6 100755 --- a/scripts/build +++ b/scripts/build @@ -439,6 +439,12 @@ done for i in $(find "${SYSROOT_PREFIX}/usr"/{lib,share} -type f -name "*.cmake" 2>/dev/null); do sed -e "s#${SYSROOT_PREFIX}/usr#${PKG_ORIG_SYSROOT_PREFIX}/usr#g" -i "${i}" done +for i in $(find "${SYSROOT_PREFIX}" -type l 2>/dev/null); do + _tmp="$(readlink -m "${i}")" + if [[ ${_tmp} =~ ^${SYSROOT_PREFIX}/ ]]; then + ln -sf "${_tmp/${SYSROOT_PREFIX}\//${PKG_ORIG_SYSROOT_PREFIX}\/}" "${i}" + fi +done # Transfer the new sysroot content to the shared sysroot mkdir -p "${PKG_ORIG_SYSROOT_PREFIX}" From 64bc8c2ab5cbd9a88f18174bfab98bf55ad31442 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 28 Feb 2019 06:19:13 +0000 Subject: [PATCH 2/7] mysql: fix sym link --- packages/databases/mysql/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/databases/mysql/package.mk b/packages/databases/mysql/package.mk index e3c05a9f9c..8b8c3bf836 100644 --- a/packages/databases/mysql/package.mk +++ b/packages/databases/mysql/package.mk @@ -79,7 +79,7 @@ post_makeinstall_target() { sed -i "s|pkgincludedir=.*|pkgincludedir=\'$SYSROOT_PREFIX/usr/include/mysql\'|" scripts/mysql_config sed -i "s|pkglibdir=.*|pkglibdir=\'$SYSROOT_PREFIX/usr/lib/mysql\'|" scripts/mysql_config cp scripts/mysql_config $SYSROOT_PREFIX/usr/bin - ln -sf $SYSROOT_PREFIX/usr/bin/mysql_config $TOOLCHAIN/bin/mysql_config + cp scripts/mysql_config $TOOLCHAIN/bin/mysql_config rm -rf $INSTALL } From ad938672b21585c7b65bb78dbf964f2c896fe5b0 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 28 Feb 2019 06:19:13 +0000 Subject: [PATCH 3/7] buildsystem: parallel pid cleanup --- config/multithread | 20 +++++++++++++------- tools/dashboard | 9 +++++++-- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/config/multithread b/config/multithread index 26a9c2ceb6..d89479ca09 100644 --- a/config/multithread +++ b/config/multithread @@ -61,7 +61,7 @@ package_worker() { . config/options "${pkgname}" - [ ! -f "${THREAD_CONTROL}/pid" ] && echo "${PARALLEL_PID}" >"${THREAD_CONTROL}/pid" + [ ! -f "${THREAD_CONTROL}/parallel.pid" ] && echo "${PARALLEL_PID}" >"${THREAD_CONTROL}/parallel.pid" ${SCRIPTS}/${task} ${pkgname} 2>&1 && result=0 || result=1 @@ -141,7 +141,7 @@ EOF export -f package_worker start_multithread_build() { - local singlethread buildopts + local singlethread buildopts result=0 # init thread control folder rm -rf "${THREAD_CONTROL}" @@ -174,12 +174,18 @@ start_multithread_build() { cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ - ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || return + ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || result=1 - cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \ - MTBUILDSTART=$(date +%s) MTWITHLOCKS=yes ${TOOLCHAIN}/bin/parallel \ - --plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" --plus ${buildopts} \ - package_worker {%} {#} {##} {} || return + if [ ${result} -eq 0 ]; then + cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \ + MTBUILDSTART=$(date +%s) MTWITHLOCKS=yes ${TOOLCHAIN}/bin/parallel \ + --plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" --plus ${buildopts} \ + package_worker {%} {#} {##} {} || result=1 + + rm -f "${THREAD_CONTROL}/parallel.pid" + fi set +o pipefail + + return ${result} } diff --git a/tools/dashboard b/tools/dashboard index efb1250948..825b00d9ed 100755 --- a/tools/dashboard +++ b/tools/dashboard @@ -10,11 +10,16 @@ if [ -n "${PROJECT}" -a "$1" != "auto" ]; then fi _find_latest_tcdir() { + local IFS=$'\n' + if [ -n "${THREAD_CONTROL}" ]; then echo "${THREAD_CONTROL}" else - ls -1tdr "${PWD}"/build.*/.threads 2>/dev/null | tail -1 + for dir in $(ls -1td "${PWD}"/build.*/.threads 2>/dev/null); do + [ -f "${dir}/parallel.pid" ] && echo "${dir}" && break + done fi + return 0 } while [ : ]; do @@ -24,7 +29,7 @@ while [ : ]; do tcdir="$(_find_latest_tcdir)" if [ -n "${tcdir}" ]; then - pid="$(cat "${tcdir}/pid" 2>/dev/null || true)" + pid="$(cat "${tcdir}/parallel.pid" 2>/dev/null || true)" [ -n "${pid}" ] && ps -p ${pid} &>/dev/null && break fi From 7b64c946cd47297d5dc46d38e1bc31bd84694230 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 1 Mar 2019 12:25:50 +0000 Subject: [PATCH 4/7] config/multithread: don't filter by PKG_ARCH --- config/multithread | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config/multithread b/config/multithread index d89479ca09..cb96bfa574 100644 --- a/config/multithread +++ b/config/multithread @@ -26,11 +26,6 @@ json_worker() { [[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local" - if [ -n "$PKG_ARCH" ]; then - listcontains "$PKG_ARCH" "!$TARGET_ARCH" && continue - listcontains "$PKG_ARCH" "$TARGET_ARCH" || listcontains "$PKG_ARCH" "any" || continue - fi - cat < Date: Sat, 2 Mar 2019 07:03:16 +0000 Subject: [PATCH 5/7] tools/viewplan: fix argument order --- tools/viewplan | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/viewplan b/tools/viewplan index 3b58af267b..09fdc889b6 100755 --- a/tools/viewplan +++ b/tools/viewplan @@ -24,6 +24,6 @@ command -v ${PARALLEL_BIN} >/dev/null || PARALLEL_BIN=fake_parallel set -o pipefail cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ - ${PARALLEL_BIN} --plain --no-notice --max-args 30 json_worker --halt now,fail=1 | \ + ${PARALLEL_BIN} --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \ die "FAILURE: Unable to generate plan" From 34670aaae75f39b74fb4503d1fd5af6ca432c5f6 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Sat, 2 Mar 2019 09:01:05 +0000 Subject: [PATCH 6/7] config/functions: slightly more efficient package cache processing --- config/functions | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/config/functions b/config/functions index c232a8a97d..6fd9ea9a4a 100644 --- a/config/functions +++ b/config/functions @@ -480,7 +480,7 @@ debug_strip() { } init_package_cache() { - local _ANCHOR="@?+?@" DIR + local _ANCHOR="@?+?@" local temp_global temp_local # If the package caches are unset, then populate them @@ -488,22 +488,16 @@ init_package_cache() { temp_global="$(mktemp)" temp_local="$(mktemp)" - # cache project/device folder for a package - if [ -n $DEVICE ]; then - for DIR in $(find $ROOT/projects/$PROJECT/devices/$DEVICE/packages -type d 2>/dev/null); do - [ -r "$DIR/package.mk" ] && echo "${DIR}${_ANCHOR}" >> "${temp_local}" - done + # cache project/device folder for packages + if [ -n "${DEVICE}" ]; then + find "${ROOT}/projects/${PROJECT}/devices/${DEVICE}/packages" -type f -name package.mk 2>/dev/null | sed "s#/package\.mk\$#${_ANCHOR}#" >> "${temp_local}" fi - # cache project folder for a package - for DIR in $(find $ROOT/projects/$PROJECT/packages -type d 2>/dev/null); do - [ -r "$DIR/package.mk" ] && echo "${DIR}${_ANCHOR}" >> "${temp_local}" - done + # cache project folder for packages + find "${ROOT}/projects/${PROJECT}/packages" -type f -name package.mk 2>/dev/null | sed "s#/package\.mk\$#${_ANCHOR}#" >> "${temp_local}" # cache packages folder - for DIR in $(find $ROOT/$PACKAGES -type d 2>/dev/null); do - [ -r "$DIR/package.mk" ] && echo "${DIR}${_ANCHOR}" >> "${temp_global}" - done + find "${ROOT}/${PACKAGES}" -type f -name package.mk 2>/dev/null | sed "s#/package\.mk\$#${_ANCHOR}#" >> "${temp_global}" _CACHE_PACKAGE_LOCAL="${BUILD}/.cache_package_local" _CACHE_PACKAGE_GLOBAL="${BUILD}/.cache_package_global" @@ -606,14 +600,14 @@ check_device() { check_arch() { local dashes="===========================" arch_err_msg linux_config_dir - if [ -d ${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/linux ]; then + if [ -d "${PROJECT_DIR}/${PROJECT}/devices/${DEVICE}/linux" ]; then linux_config_dir="${PROJECT_DIR}/${PROJECT}/devices/$DEVICE/linux" else linux_config_dir="${PROJECT_DIR}/${PROJECT}/linux" fi - if [ ! -e $linux_config_dir/linux.${TARGET_PATCH_ARCH:-$TARGET_ARCH}.conf ] && - ! ls $linux_config_dir/*/linux.${TARGET_PATCH_ARCH:-$TARGET_ARCH}.conf &>/dev/null; then + if [ ! -e "$linux_config_dir/linux.${TARGET_PATCH_ARCH:-$TARGET_ARCH}.conf" ] && + ! ls "$linux_config_dir/"*/linux.${TARGET_PATCH_ARCH:-$TARGET_ARCH}.conf &>/dev/null; then arch_err_msg="\n $dashes$dashes$dashes" arch_err_msg="${arch_err_msg}\n ERROR: Architecture not found, use a valid Architecture" arch_err_msg="${arch_err_msg}\n for your project or create a new config" From a8572849b0b66c06d3fa0086955d1ce58ac6a79a Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Mon, 4 Mar 2019 11:58:16 +0000 Subject: [PATCH 7/7] scripts/build_mt: add checkdeps --- scripts/build_mt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/build_mt b/scripts/build_mt index 5e3dbfc436..d7b35168df 100755 --- a/scripts/build_mt +++ b/scripts/build_mt @@ -6,6 +6,8 @@ . config/options "" . config/multithread +${SCRIPTS}/checkdeps + # Setup both toolchain cmake configs to avoid potentially racy behaviour later. # Use a fork for host to isolate any variable modifications. ( setup_toolchain host )