From 527ea323b8de0143638c8c3c62f9a4c2cdb11d63 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Tue, 19 Feb 2019 19:03:58 +0000 Subject: [PATCH 01/12] scripts/build: a shared sysroot is not thread-safe --- scripts/build | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/scripts/build b/scripts/build index c123a5b8b9..31973ed89a 100755 --- a/scripts/build +++ b/scripts/build @@ -373,13 +373,14 @@ fi pkg_call_exists post_make_$TARGET && pkg_call post_make_$TARGET -# make install writes directly to sysroot which then needs fixing up - this is racy -if listcontains "configure:target cmake-make:target autotools:target make:target" "${PKG_TOOLCHAIN}:${TARGET}"; then - acquire_exclusive_lock "${PKG_NAME}:${TARGET}" "build" - PKG_NEED_UNLOCK=yes -else - PKG_NEED_UNLOCK=no -fi +# Hack around directly writing/modifying the content of a shared sysroot +# by temporarily installing new files to a package specific sysroot +PKG_ORIG_SYSROOT_PREFIX="${SYSROOT_PREFIX}" +export SYSROOT_PREFIX="${BUILD}/.sysroot/${PKG_NAME}.${TARGET}" +rm -rf "${SYSROOT_PREFIX}" +for d in /usr/lib /usr/include /usr/bin /usr/lib/pkgconfig; do + mkdir -p "${SYSROOT_PREFIX}${d}" +done # make install pkg_call_exists pre_makeinstall_$TARGET && pkg_call pre_makeinstall_$TARGET @@ -422,14 +423,26 @@ fi pkg_call_exists post_makeinstall_$TARGET && pkg_call post_makeinstall_$TARGET -# This is the racy part - one or more processes may be creating, modifying or deleting -# $SYSROOT_PREFIX/usr/lib/*.la files while another (eg. this) process is rewriting the same -# files as it fixes up the libdir prefix. Eugh. -for i in $(find $SYSROOT_PREFIX/usr/lib/ -name "*.la" 2>/dev/null); do - sed -e "s:\(['= ]\)/usr:\\1$SYSROOT_PREFIX/usr:g" -i $i +# Fixup temporary sysroot references to the shared sysroot +for i in $(find "${SYSROOT_PREFIX}/usr/lib" -type f -name "*.la" 2>/dev/null); do + sed -e "s:\(['= ]\)/usr:\\1${PKG_ORIG_SYSROOT_PREFIX}/usr:g" -i "${i}" +done +for i in $(find "${SYSROOT_PREFIX}/usr/bin" -type f -name "*-config" 2>/dev/null); do + sed -e "s#${SYSROOT_PREFIX}/usr#${PKG_ORIG_SYSROOT_PREFIX}/usr#g" -i "${i}" +done +for i in $(find "${SYSROOT_PREFIX}/usr/lib" -type f -name "*.pc" 2>/dev/null); do + sed -e "s#${SYSROOT_PREFIX}/usr#${PKG_ORIG_SYSROOT_PREFIX}/usr#g" -i "${i}" +done +for i in $(find "${SYSROOT_PREFIX}/usr"/{lib,share} -type f -name "*.cmake" 2>/dev/null); do + sed -e "s#${SYSROOT_PREFIX}/usr#${PKG_ORIG_SYSROOT_PREFIX}/usr#g" -i "${i}" done -[ "${PKG_NEED_UNLOCK}" = "yes" ] && release_exclusive_lock "${PKG_NAME}:${TARGET}" "build" +# Transfer the new sysroot content to the shared sysroot +mkdir -p "${PKG_ORIG_SYSROOT_PREFIX}" +cp -PR "${SYSROOT_PREFIX}"/* "${PKG_ORIG_SYSROOT_PREFIX}" +rm -rf "${SYSROOT_PREFIX}" + +export SYSROOT_PREFIX="${PKG_ORIG_SYSROOT_PREFIX}" if [ "$TARGET" = "target" -o "$TARGET" = "init" ]; then if [ -d $INSTALL ]; then From f395e17adb533893ead02d69b95ca0da077a461a Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 21 Feb 2019 01:43:07 +0000 Subject: [PATCH 02/12] libimobiledevice: ensure directory exists before copy --- packages/addons/addon-depends/libimobiledevice/package.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/addons/addon-depends/libimobiledevice/package.mk b/packages/addons/addon-depends/libimobiledevice/package.mk index 08e409426e..62e040e684 100644 --- a/packages/addons/addon-depends/libimobiledevice/package.mk +++ b/packages/addons/addon-depends/libimobiledevice/package.mk @@ -17,5 +17,6 @@ PKG_CONFIGURE_OPTS_TARGET="--enable-static \ --disable-largefile" post_makeinstall_target() { - cp $PKG_BUILD/common/utils.h $SYSROOT_PREFIX/usr/include/libimobiledevice/ + mkdir -p "${SYSROOT_PREFIX}/usr/include/lib/libimobiledevice" + cp ${PKG_BUILD}/common/utils.h "${SYSROOT_PREFIX}/usr/include/libimobiledevice" } From a1ecb7099196acb78ec513bd5c3951027bcb6099 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Wed, 20 Feb 2019 05:05:08 +0000 Subject: [PATCH 03/12] pvr.zattoo: missing dependency --- packages/mediacenter/kodi-binary-addons/pvr.zattoo/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediacenter/kodi-binary-addons/pvr.zattoo/package.mk b/packages/mediacenter/kodi-binary-addons/pvr.zattoo/package.mk index e73dcdbca4..ec8fa6430c 100644 --- a/packages/mediacenter/kodi-binary-addons/pvr.zattoo/package.mk +++ b/packages/mediacenter/kodi-binary-addons/pvr.zattoo/package.mk @@ -9,7 +9,7 @@ PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="http://www.kodi.tv" PKG_URL="https://github.com/rbuehlma/pvr.zattoo/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain kodi-platform" +PKG_DEPENDS_TARGET="toolchain kodi-platform tinyxml2" PKG_SECTION="" PKG_SHORTDESC="pvr.zattoo" PKG_LONGDESC="pvr.zattoo" From ff567d405b48833e649120f1c315e40ab5f0d6d8 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Wed, 20 Feb 2019 20:20:32 +0000 Subject: [PATCH 04/12] libopenmpt: missing dependency --- packages/audio/libopenmpt/package.mk | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/audio/libopenmpt/package.mk b/packages/audio/libopenmpt/package.mk index cad60b79db..af38eb6167 100644 --- a/packages/audio/libopenmpt/package.mk +++ b/packages/audio/libopenmpt/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="libopenmpt" PKG_VERSION="0.3.1" @@ -7,7 +8,7 @@ PKG_SHA256="6fc5614926edd523d5585e40fdaf0b2ea08689d4dc91de49511d02503744cbb5" PKG_LICENSE="BSD" PKG_SITE="http://lib.openmpt.org/libopenmpt/" PKG_URL="http://lib.openmpt.org/files/libopenmpt/src/${PKG_NAME}-${PKG_VERSION}+release.autotools.tar.gz" -PKG_DEPENDS_TARGET="toolchain libogg libvorbis" +PKG_DEPENDS_TARGET="toolchain libogg libvorbis zlib" PKG_LONGDESC="libopenmpt renders mod music files as raw audio data, for playing or conversion." PKG_BUILD_FLAGS="+pic" From 7747d0aaeaf439e07bf64c704d8ecb46c5eaa256 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 21 Feb 2019 03:05:59 +0000 Subject: [PATCH 05/12] qtbase: meh, fix itself up --- .../addons/addon-depends/qtbase/package.mk | 62 +++++++++++-------- 1 file changed, 37 insertions(+), 25 deletions(-) diff --git a/packages/addons/addon-depends/qtbase/package.mk b/packages/addons/addon-depends/qtbase/package.mk index c8a6583281..a6ee847c25 100644 --- a/packages/addons/addon-depends/qtbase/package.mk +++ b/packages/addons/addon-depends/qtbase/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Lukas Rusak (lrusak@libreelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="qtbase" PKG_VERSION="5.6.2" @@ -11,8 +12,8 @@ PKG_DEPENDS_TARGET="pcre zlib" PKG_LONGDESC="A cross-platform application and UI framework." PKG_CONFIGURE_OPTS_TARGET="-prefix /usr - -sysroot $SYSROOT_PREFIX - -hostprefix $TOOLCHAIN + -sysroot "${SYSROOT_PREFIX}" + -hostprefix "${TOOLCHAIN}" -device linux-libreelec-g++ -opensource -confirm-license -release @@ -52,31 +53,42 @@ PKG_CONFIGURE_OPTS_TARGET="-prefix /usr configure_target() { QMAKE_CONF_DIR="mkspecs/devices/linux-libreelec-g++" - QMAKE_CONF="${QMAKE_CONF_DIR}/qmake.conf" cd .. - mkdir -p $QMAKE_CONF_DIR - echo "MAKEFILE_GENERATOR = UNIX" > $QMAKE_CONF - echo "CONFIG += incremental" >> $QMAKE_CONF - echo "QMAKE_INCREMENTAL_STYLE = sublib" >> $QMAKE_CONF - echo "include(../../common/linux.conf)" >> $QMAKE_CONF - echo "include(../../common/gcc-base-unix.conf)" >> $QMAKE_CONF - echo "include(../../common/g++-unix.conf)" >> $QMAKE_CONF - echo "load(device_config)" >> $QMAKE_CONF - echo "QMAKE_CC = $CC" >> $QMAKE_CONF - echo "QMAKE_CXX = $CXX" >> $QMAKE_CONF - echo "QMAKE_LINK = $CXX" >> $QMAKE_CONF - echo "QMAKE_LINK_SHLIB = $CXX" >> $QMAKE_CONF - echo "QMAKE_AR = $AR cqs" >> $QMAKE_CONF - echo "QMAKE_OBJCOPY = $OBJCOPY" >> $QMAKE_CONF - echo "QMAKE_NM = $NM -P" >> $QMAKE_CONF - echo "QMAKE_STRIP = $STRIP" >> $QMAKE_CONF - echo "QMAKE_CFLAGS = $CFLAGS" >> $QMAKE_CONF - echo "QMAKE_CXXFLAGS = $CXXFLAGS" >> $QMAKE_CONF - echo "QMAKE_LFLAGS = $LDFLAGS" >> $QMAKE_CONF - echo "load(qt_config)" >> $QMAKE_CONF - echo '#include "../../linux-g++/qplatformdefs.h"' >> $QMAKE_CONF_DIR/qplatformdefs.h + mkdir -p ${QMAKE_CONF_DIR} + + cat >"${QMAKE_CONF_DIR}/qmake.conf" <"${QMAKE_CONF_DIR}/qplatformdefs.h" < Date: Thu, 21 Feb 2019 18:20:27 +0000 Subject: [PATCH 06/12] screensaver.asterwave: missing dependency --- .../kodi-binary-addons/screensaver.asterwave/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediacenter/kodi-binary-addons/screensaver.asterwave/package.mk b/packages/mediacenter/kodi-binary-addons/screensaver.asterwave/package.mk index 5d36c71e75..c147866828 100644 --- a/packages/mediacenter/kodi-binary-addons/screensaver.asterwave/package.mk +++ b/packages/mediacenter/kodi-binary-addons/screensaver.asterwave/package.mk @@ -10,7 +10,7 @@ PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="https://github.com/notspiff/screensaver.asterwave" PKG_URL="https://github.com/notspiff/screensaver.asterwave/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain kodi-platform soil" +PKG_DEPENDS_TARGET="toolchain kodi-platform soil glm" PKG_SECTION="" PKG_SHORTDESC="screensaver.asterwave" PKG_LONGDESC="screensaver.asterwave" From 94b9196ebc8a1641798f840d5d8b7aa14f1b14ec Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 21 Feb 2019 18:20:43 +0000 Subject: [PATCH 07/12] screensaver.cpblobs: missing dependency --- .../kodi-binary-addons/screensaver.cpblobs/package.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/mediacenter/kodi-binary-addons/screensaver.cpblobs/package.mk b/packages/mediacenter/kodi-binary-addons/screensaver.cpblobs/package.mk index 673b75e049..39f3a85873 100644 --- a/packages/mediacenter/kodi-binary-addons/screensaver.cpblobs/package.mk +++ b/packages/mediacenter/kodi-binary-addons/screensaver.cpblobs/package.mk @@ -10,7 +10,7 @@ PKG_ARCH="any" PKG_LICENSE="GPL" PKG_SITE="https://github.com/notspiff/screensaver.cpblobs" PKG_URL="https://github.com/notspiff/screensaver.cpblobs/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain kodi-platform soil" +PKG_DEPENDS_TARGET="toolchain kodi-platform soil glm" PKG_SECTION="" PKG_SHORTDESC="screensaver.cpblobs" PKG_LONGDESC="screensaver.cpblobs" From 00cc78ea68214312e7d458d38dedcb7d25d93645 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 22 Feb 2019 19:34:36 +0000 Subject: [PATCH 08/12] scripts/genbuildplan.py: typo --- scripts/genbuildplan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py index 73a5d175d5..3922f886a7 100755 --- a/scripts/genbuildplan.py +++ b/scripts/genbuildplan.py @@ -313,7 +313,7 @@ def processPackages(args, packages, build): elif not args.ignore_invalid: raise Exception("Invalid package! Package %s cannot be found for this PROJECT/DEVICE/ARCH" % dfq_p) - # To each target-specific node, add the correspnding + # To each target-specific node, add the corresponding # target-specific dependency nodes ("edges") for name in node_map: node = node_map[name] From 3f3395fdf4992d907ccdfaf4bd2d5b3a8c66ebf3 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 22 Feb 2019 20:08:42 +0000 Subject: [PATCH 09/12] buildsystem: switch rm -fr to rm -rf for consistency/OCD --- config/multithread | 2 +- scripts/unpack | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/multithread b/config/multithread index 1c14ddfc70..26a9c2ceb6 100644 --- a/config/multithread +++ b/config/multithread @@ -144,7 +144,7 @@ start_multithread_build() { local singlethread buildopts # init thread control folder - rm -fr "${THREAD_CONTROL}" + rm -rf "${THREAD_CONTROL}" mkdir -p "${THREAD_CONTROL}/locks" echo -1 >"${THREAD_CONTROL}/progress.prev" echo 0 >"${THREAD_CONTROL}/progress" diff --git a/scripts/unpack b/scripts/unpack index 1fc760aefc..5415acf5aa 100755 --- a/scripts/unpack +++ b/scripts/unpack @@ -54,7 +54,7 @@ if [ -d "$SOURCES/${PKG_NAME}" -o -d "$PKG_DIR/sources" ]; then # unpack into a unique location as unpacking into a single ${BUILD} directory is not thread-safe PKG_UNPACK_DIR="${BUILD}/.unpack/${PKG_NAME}" - rm -fr "${PKG_UNPACK_DIR}" + rm -rf "${PKG_UNPACK_DIR}" mkdir -p "${PKG_UNPACK_DIR}" pkg_call_exists pre_unpack && pkg_call pre_unpack @@ -84,7 +84,7 @@ if [ -d "$SOURCES/${PKG_NAME}" -o -d "$PKG_DIR/sources" ]; then fi # cleanup - rm -fr "${PKG_UNPACK_DIR}" + rm -rf "${PKG_UNPACK_DIR}" if [ -d "$PKG_DIR/sources" ]; then [ ! -d "$PKG_BUILD" ] && mkdir -p "$PKG_BUILD" From 83922eef4f3d9c7b3f8b13f5573dcc6af7b4ea6a Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 22 Feb 2019 20:11:11 +0000 Subject: [PATCH 10/12] scripts/build: add TODO --- scripts/build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/build b/scripts/build index 31973ed89a..a4be68d83f 100755 --- a/scripts/build +++ b/scripts/build @@ -378,6 +378,9 @@ pkg_call_exists post_make_$TARGET && pkg_call post_make_$TARGET PKG_ORIG_SYSROOT_PREFIX="${SYSROOT_PREFIX}" export SYSROOT_PREFIX="${BUILD}/.sysroot/${PKG_NAME}.${TARGET}" rm -rf "${SYSROOT_PREFIX}" + +# Create common sysroot directories as some packages expect them to exist. +# TODO: Fix those packages so we don't need to pre-create directories. for d in /usr/lib /usr/include /usr/bin /usr/lib/pkgconfig; do mkdir -p "${SYSROOT_PREFIX}${d}" done From 956b10dd5564d5f03beacc5ef1c3e0ed90fa1ad8 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Mon, 25 Feb 2019 20:29:34 +0000 Subject: [PATCH 11/12] scripts/image: restore checkdeps call --- scripts/image | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/image b/scripts/image index 70830e2050..ba9b1aa9e3 100755 --- a/scripts/image +++ b/scripts/image @@ -13,6 +13,8 @@ unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACK show_config save_build_config +${SCRIPTS}/checkdeps + # Setup both toolchain cmake configs to avoid potentially racy behaviour later. # Use a fork for host to isolate any variable modifications. ( setup_toolchain host ) From 6da44a393103e4c988ef7e90565a8506994e0ae9 Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Mon, 25 Feb 2019 20:59:12 +0000 Subject: [PATCH 12/12] config/functions: maybe pedantic, but just in case append isn't thread-safe --- config/functions | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/config/functions b/config/functions index 050498b604..c232a8a97d 100644 --- a/config/functions +++ b/config/functions @@ -1261,11 +1261,15 @@ pkg_lock_status() { local status="$1" pkg="$2" task="$3" msg="$4" local this_job="${MTJOBID}" line - printf -v line "%s: <%05d> [%02d/%0*d] %-7s %-7s %-35s" \ - "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" - [ -n "${msg}" ] && line+=" (${msg})" + ( + flock --exclusive 94 - echo "${line}" >>"${THREAD_CONTROL}/history" + printf -v line "%s: <%05d> [%02d/%0*d] %-7s %-7s %-35s" \ + "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" + [ -n "${msg}" ] && line+=" (${msg})" + + echo "${line}" >>"${THREAD_CONTROL}/history" + ) 94>"${THREAD_CONTROL}/locks/.history" [ "${DASHBOARD}" != "no" ] && update_dashboard "${status}" "${pkg}" "${task}" "${msg}"