diff --git a/Makefile b/Makefile index a26a9e2803..c5229b6449 100644 --- a/Makefile +++ b/Makefile @@ -17,6 +17,22 @@ noobs: amlpkg: ./scripts/image amlpkg +# legacy sequential build targets +system-st: + ./scripts/image_st + +release-st: + ./scripts/image_st release + +image-st: + ./scripts/image_st mkimage + +noobs-st: + ./scripts/image_st noobs + +amlpkg-st: + ./scripts/image_st amlpkg + clean: rm -rf $(BUILD_DIRS)/* $(BUILD_DIRS)/.stamps diff --git a/config/functions b/config/functions index c7012473ee..afd0a72f3b 100644 --- a/config/functions +++ b/config/functions @@ -857,7 +857,7 @@ source_package() { unset_functions if [ -n "${1}" ]; then - PKG_DIR="$(get_pkg_directory ${1})" + [ -f "${1}" ] && PKG_DIR="$(dirname "${1}")" || PKG_DIR="$(get_pkg_directory "${1}")" [ -n "$PKG_DIR" -a -r $PKG_DIR/package.mk ] || die "FAILURE: unable to source package - ${1}/package.mk does not exist" @@ -873,6 +873,12 @@ source_package() { [ "$PKG_ADDON_IS_STANDALONE" != "yes" ] && PKG_NEED_UNPACK="${PKG_NEED_UNPACK} $(get_pkg_directory $MEDIACENTER)" fi + if [ -n "${PKG_DEPENDS_UNPACK}" ]; then + for _p in ${PKG_DEPENDS_UNPACK}; do + PKG_NEED_UNPACK+=" $(get_pkg_directory ${_p})" + done + fi + # Automatically set PKG_SOURCE_NAME unless it is already defined. # PKG_SOURCE_NAME will be automatically set to a name based on # the $PKG_NAME-$PKG_VERSION convention. @@ -1175,7 +1181,7 @@ add_group() { } # Usage: enable_service [target] -enable_service () { +enable_service() { local unit="$1" local unit_dir="/usr/lib/systemd/system" local target="$2" @@ -1197,6 +1203,174 @@ enable_service () { done } + +### MULTI-THREADED FUNCTION HELPERS ### +# Prevent concurrent modifications to a package (unpack) or +# package:target (install/build). +# +# If a package is already locked and the owner is ourselves +# then assume we already have the required lock. +pkg_lock() { + [ "${MTWITHLOCKS}" != "yes" ] && return 0 + + local pkg="$1" task="$2" parent_pkg="$3" + local this_job="${MTJOBID}" + local lock_job lock_task lock_pkg + local fail_seq + + exec 98>"${THREAD_CONTROL}/locks/${pkg}.${task}" + if ! flock --nonblock --exclusive 98; then + while [ : ]; do + read -r lock_job lock_task lock_pkg <<<$(cat "${THREAD_CONTROL}/locks/${pkg}.${task}.owner" 2>/dev/null) + [ -n "${lock_job}" ] && break || sleep 1 + done + + if [ ${lock_job} != ${this_job} ]; then + pkg_lock_status "STALLED" "${parent_pkg}" "${task}" "$(printf "waiting on [%02d] %s %s" ${lock_job} "${lock_task}" "${lock_pkg}")" + flock --exclusive 98 + fi + fi + + # As we now have the lock, if .failed still exists then a previous process must have failed + if [ -f "${THREAD_CONTROL}/locks/${pkg}.${task}.failed" ]; then + fail_seq="$(cat "${THREAD_CONTROL}/locks/${pkg}.${task}.failed")" + print_color CLR_ERROR "FAILURE: ${pkg}.${task}.failed exists, a previous dependency process has failed (seq: ${fail_seq})\n" + if [ -d "${THREAD_CONTROL}/logs" ]; then +cat <"${THREAD_CONTROL}/locks/${pkg}.${task}.owner" + + pkg_lock_status "LOCKED" "${pkg}" "${task}" +} + +# Log additional information for a locked package. +pkg_lock_status() { + [ "${MTWITHLOCKS}" != "yes" ] && return 0 + + local status="$1" pkg="$2" task="$3" msg="$4" + local this_job="${MTJOBID}" line + + printf -v line "%s: <%05d> [%02d/%0*d] %-7s %-7s %-35s" \ + "$(date +%Y-%m-%d\ %H:%M:%S.%N)" $$ ${this_job} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${status}" "${task}" "${pkg}" + [ -n "${msg}" ] && line+=" (${msg})" + + echo "${line}" >>"${THREAD_CONTROL}/history" + + [ "${DASHBOARD}" != "no" ] && update_dashboard "${status}" "${pkg}" "${task}" "${msg}" + + if [ "${status}" = "LOCKED" ]; then + echo "${PARALLEL_SEQ}" > "${THREAD_CONTROL}/locks/${pkg}.${task}.failed" + elif [ "${status}" = "UNLOCK" ]; then + rm "${THREAD_CONTROL}/locks/${pkg}.${task}.failed" + fi + + return 0 +} + +update_dashboard() { + local status="$1" pkg="$2" task="$3" msg="$4" + local line sedline preamble num elapsed projdevarch + local boldred boldgreen boldyellow endcolor + + ( + flock --exclusive 97 + + [ -n "${MTJOBID}" ] && sedline=$((MTJOBID + 2)) || sedline=1 + + num=$(cat "${THREAD_CONTROL}/status" | wc -l) + while [ ${num} -lt ${sedline} ]; do echo "" >>"${THREAD_CONTROL}/status"; num=$((num + 1)); done + + num=$(($(cat "${THREAD_CONTROL}/progress.prev") + 1)) + projdevarch="${PROJECT}/" + [ -n "${DEVICE}" ] && projdevarch+="${DEVICE}/" + projdevarch+="${TARGET_ARCH}" + TZ=UTC0 printf -v elapsed "%(%H:%M:%S)T" $(($(date +%s) - MTBUILDSTART)) + printf -v preamble "%s Dashboard (%s) - %d of %d jobs completed, %s elapsed" "${DISTRONAME}" "${projdevarch}" ${num} ${MTMAXJOBS} "${elapsed}" + printf -v preamble "%b%-105s %s" "\e[2J\e[0;0H" "${preamble//\//\\/}" "$(date "+%Y-%m-%d %H:%M:%S")" + + # Only update the header when caller is not a worker thread + if [ -z "${MTJOBID}" ]; then + sed -e "1s/.*/${preamble}/" -i "${THREAD_CONTROL}/status" + else + if [ "${DISABLE_COLORS}" != "yes" ]; then + boldred="\e[1;31m" + boldgreen="\e[1;32m" + boldyellow="\e[1;33m" + white="\e[0;37m" + endcolor="\e[0m" + + case "${status}" in + IDLE) color="${white}";; + STALLED) color="${boldyellow}";; + MUTEX/W) color="${boldyellow}";; + FAILED ) color="${boldred}";; + *) color="${boldgreen}";; + esac + fi + + printf -v line "[%02d\/%0*d] %b%-7s%b %-7s %-35s" ${MTJOBID} ${#MTMAXJOBS} ${PARALLEL_SEQ:-0} "${color}" "${status//\//\\/}" "${endcolor}" "${task}" "${pkg}" + [ -n "${msg}" ] && line+=" ${msg//\//\\/}" + sed -e "1s/.*/${preamble}/;${sedline}s/.*/${line}/" -i "${THREAD_CONTROL}/status" + fi + ) 97>"${THREAD_CONTROL}/locks/.status" +} + +# Thread concurrency helpers to avoid concurrency issues with some code, +# eg. when Python installs directly into $TOOLCHAIN. +# Test MTJOBID so that these functions are a no-op during non-multithreaded builds. +acquire_exclusive_lock() { + [ "${MTWITHLOCKS}" != "yes" ] && return 0 + + local pkg="$1" task="$2" lockfile="${3:-global}" + local this_job="${MTJOBID}" + local lock_job lock_task lock_pkg + + exec 96>"${THREAD_CONTROL}/locks/.mutex.${lockfile}" + if ! flock --nonblock --exclusive 96; then + while [ : ]; do + read -r lock_job lock_task lock_pkg <<<$(cat "${THREAD_CONTROL}/locks/.mutex.${lockfile}.owner" 2>/dev/null) + [ -n "${lock_job}" ] && break || sleep 1 + done + + if [ ${lock_job} != ${this_job} ]; then + pkg_lock_status "MUTEX/W" "${pkg}" "${task}" "$(printf "mutex: %s; waiting on [%02d] %s %s" "${lockfile}" ${lock_job} "${lock_task}" "${lock_pkg}")" + flock --exclusive 96 + fi + fi + echo "${this_job} ${task} ${pkg}" >"${THREAD_CONTROL}/locks/.mutex.${lockfile}.owner" + + pkg_lock_status "MUTEX" "${pkg}" "${task}" "mutex: ${lockfile}" +} + +release_exclusive_lock() { + [ "${MTWITHLOCKS}" != "yes" ] && return 0 + + local pkg="$1" task="$2" + + flock --unlock 96 2>/dev/null + + pkg_lock_status "ACTIVE" "${pkg}" "${task}" +} + +# Execute single command using mutex +exec_thread_safe() { + local result + acquire_exclusive_lock "${PKG_NAME:exec}" "execcmd" + $@ + result=$? + release_exclusive_lock "${PKG_NAME:exec}" "execcmd" + return ${result} +} + # Use distribution functions if any if [ -f "distributions/$DISTRO/config/functions" ]; then . distributions/$DISTRO/config/functions diff --git a/config/multithread b/config/multithread new file mode 100644 index 0000000000..2ffa4c6dda --- /dev/null +++ b/config/multithread @@ -0,0 +1,183 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +THREADCOUNT=${THREADCOUNT:-100%} + +# This function is passed a list of package.mk paths to be processed. +# Each package.mk is sourced with relevant variables output in JSON format. +json_worker() { + local packages="$@" + local pkgpath hierarchy exited + + exit() { exited=1; } + + . config/options "" + + for pkgpath in ${packages}; do + pkgpath="${pkgpath%%@*}" + + exited=0 + if ! source_package "${pkgpath}/package.mk" &>/dev/null; then + unset -f exit + die "$(print_color CLR_ERROR "FAILURE: sourcing package ${pkgpath}/package.mk")" + fi + + [ ${exited} -eq 1 ] && continue + + [[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local" + + if [ -n "$PKG_ARCH" ]; then + listcontains "$PKG_ARCH" "!$TARGET_ARCH" && continue + listcontains "$PKG_ARCH" "$TARGET_ARCH" || listcontains "$PKG_ARCH" "any" || continue + fi + + cat <" or "install ". +# ${slot} is the job slot number, ie. 1-8 when THREADCOUNT=8. +# ${job} is the sequence within the total number of ${jobs}. +package_worker() { + local slot=$1 job=$2 jobs=$3 args="$4" + local task pkgname result status + local addon istarget + + export MTJOBID=${slot} MTMAXJOBS=${jobs} + + read -r task pkgname <<< "${args}" + + . config/options "${pkgname}" + + [ ! -f "${THREAD_CONTROL}/pid" ] && echo "${PARALLEL_PID}" >"${THREAD_CONTROL}/pid" + + ${SCRIPTS}/${task} ${pkgname} 2>&1 && result=0 || result=1 + + [[ ${pkgname} =~ :target$ || "${pkgname//:/}" = "${pkgname}" ]] && istarget="yes" || istarget="no" + + if [ "${MTADDONBUILD}" = "yes" -a "${PKG_IS_ADDON}" = "yes" -a "${istarget}" = "yes" ]; then + if [ ${result} -eq 0 ]; then + ( + pkg_lock "${pkgname}" "packadd" + pkg_lock_status "ACTIVE" "${pkgname}" "packadd" + + # cleanup old install path + rm -rf "${ADDON_BUILD}" + + # install addon parts + if pkg_call_exists addon; then + pkg_call addon + else + install_binary_addon "${PKG_ADDON_ID}" + fi + + # HACK for packages that provide multiple addons like screensavers.rsxs + # addon's addon() in package.mk should take care for exporting + # MULTI_ADDONS="addon.boo1 addon.boo2 addon.boo3" + if [ -n "${MULTI_ADDONS}" ] ; then + for addon in ${MULTI_ADDONS}; do + ${SCRIPTS}/install_addon "${PKG_NAME}" "${addon}" + done + else + ${SCRIPTS}/install_addon "${PKG_NAME}" "${PKG_ADDON_ID}" + fi + + pkg_lock_status "UNLOCK" "${pkgname}" "packadd" "packed" + ) 2>&1 || result=1 + fi + + if [ ${result} -ne 0 ]; then + if [ -d "${THREAD_CONTROL}/logs" ]; then + echo "${PKG_NAME} ${THREAD_CONTROL}/logs/${job}/stdout" >>"${THREAD_CONTROL}/addons.failed" + else + echo "${PKG_NAME}" >>"${THREAD_CONTROL}/addons.failed" + fi + fi + fi + + ( + flock --exclusive 95 + [ ${result} -eq 0 ] && status="DONE" || status="FAIL" + num=$(($(cat "${THREAD_CONTROL}/progress") + 1)) + mv "${THREAD_CONTROL}/progress" "${THREAD_CONTROL}/progress.prev" + echo ${num} >"${THREAD_CONTROL}/progress" + printf "[%0*d/%0*d] [%-4s] %-7s %s\n" ${#jobs} ${num} ${#jobs} ${jobs} "${status}" "${task}" "${pkgname}" >&2 + ) 95>"${THREAD_CONTROL}/locks/.progress" + + if [ ${result} -eq 0 ]; then + pkg_lock_status "IDLE" + else + pkg_lock_status "FAILED" "${pkgname}" "${task}" + + print_color CLR_ERROR "FAILURE: $SCRIPTS/${task} ${pkgname} has failed!\n" + + if [ -d "${THREAD_CONTROL}/logs" ]; then + cat >&2 <"${THREAD_CONTROL}/progress.prev" + echo 0 >"${THREAD_CONTROL}/progress" + touch "${THREAD_CONTROL}/status" + + [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1 + + # Bootstrap GNU parallel + $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package" + + # if number of detected slots is 1 then don't bother using inter-process locks as this is a sequential build + [ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && singlethread=yes || singlethread=no + + # create a single log file by default if not using locks (single build process), or the builder is a masochist + if [ "${singlethread}" = "yes" -a "${ONELOG,,}" != "no" ] || [ "${ONELOG,,}" = "yes" ]; then + buildopts+=" --ungroup" + else + mkdir -p "${THREAD_CONTROL}/logs" + buildopts+=" --group --results ${THREAD_CONTROL}/logs/{#}/" + fi + + # When building addons, don't halt on error - keep building all packages/addons + [ "${MTADDONBUILD}" = "yes" ] && buildopts+=" --halt never" || buildopts+=" --halt now,fail=1" + + # pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status + set -o pipefail + + cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ + ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ + ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || return + + cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \ + MTBUILDSTART=$(date +%s) MTWITHLOCKS=yes ${TOOLCHAIN}/bin/parallel \ + --plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" --plus ${buildopts} \ + package_worker {%} {#} {##} {} || return + + set +o pipefail +} diff --git a/config/path b/config/path index 1f1080b53c..fd0dddc544 100644 --- a/config/path +++ b/config/path @@ -29,6 +29,8 @@ if [ -n "$BUILD_SUFFIX" ]; then BUILD=$BUILD-$BUILD_SUFFIX fi +THREAD_CONTROL=${BUILD}/.threads + TARGET_ADDONS="$TARGET_IMG/$ADDONS/$ADDON_PATH" ADDON_BUILD="$BUILD/$ADDONS/$1" STAMPS_NOARCH=.stamps @@ -60,7 +62,6 @@ INSTALL_INIT=$BUILD/image/initramfs/root-image . config/sources MAKE="$TOOLCHAIN/bin/make" -MAKEINSTALL="$TOOLCHAIN/bin/make -j1 DESTDIR=$SYSROOT_PREFIX install" XORG_PATH_DRI=/usr/lib/dri XORG_PATH_XKB=/usr/share/X11/xkb @@ -82,18 +83,16 @@ fi VERSION_SUFFIX=$TARGET_ARCH # redirect formatted output -if [ -z "${SILENT_OUT}" -a -z "${VERBOSE_OUT}" ]; then - export BUILD_INDENT_SIZE=4 - export SILENT_OUT=3 - export VERBOSE_OUT=4 +export BUILD_INDENT_SIZE=4 +SILENT_OUT=3 +VERBOSE_OUT=4 - if [ "$VERBOSE" = yes ]; then - exec 3>&1 - exec 4>&2 - else - exec 3>&2 - exec 4>/dev/null - fi +if [ "$VERBOSE" = yes ]; then + exec 3>&1 + exec 4>&1 +else + exec 3>&2 + exec 4>/dev/null fi unset LD_LIBRARY_PATH diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/package.mk index dc65abc96d..7d851bc0e3 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXcomposite/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXcomposite" PKG_LONGDESC="libXcomposite for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXcomposite" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/package.mk index 6c7525891e..f79d9987ab 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXdamage/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXdamage" PKG_LONGDESC="libXdamage for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXdamage" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/package.mk index e373ff806a..fc1725a861 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXfixes/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXfixes" PKG_LONGDESC="libXfixes for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXfixes" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXi/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXi/package.mk index 91281c1018..d75a7e62b8 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXi/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXi/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXi" PKG_LONGDESC="libXi for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXi" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXrender/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXrender/package.mk index 2d91dba614..66eb1e3ac0 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXrender/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXrender/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXrender" PKG_LONGDESC="libXrender for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXrender" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libXtst/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libXtst/package.mk index dfe0bfc12c..01060424ab 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libXtst/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libXtst/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libXtst" PKG_LONGDESC="libXtst for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libXtst" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/chrome-depends/chrome-libxcb/package.mk b/packages/addons/addon-depends/chrome-depends/chrome-libxcb/package.mk index a9caed23d1..1fe4b9ebd7 100644 --- a/packages/addons/addon-depends/chrome-depends/chrome-libxcb/package.mk +++ b/packages/addons/addon-depends/chrome-depends/chrome-libxcb/package.mk @@ -6,6 +6,7 @@ PKG_NAME="chrome-libxcb" PKG_LONGDESC="libxcb for chrome" PKG_URL="" +PKG_DEPENDS_UNPACK+=" libxcb" PKG_CONFIGURE_OPTS_TARGET="$PKG_CONFIGURE_OPTS_TARGET \ --disable-static \ diff --git a/packages/addons/addon-depends/cxxtools/package.mk b/packages/addons/addon-depends/cxxtools/package.mk index f7f31e898c..2af05c5295 100644 --- a/packages/addons/addon-depends/cxxtools/package.mk +++ b/packages/addons/addon-depends/cxxtools/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="cxxtools" PKG_VERSION="2.2.1" @@ -7,8 +8,8 @@ PKG_SHA256="8cebb6d6cda7c93cc4f7c0d552a68d50dd5530b699cf87916bb3b708fdc4e342" PKG_LICENSE="GPL-2" PKG_SITE="http://www.tntnet.org/cxxtools.html" PKG_URL="http://www.tntnet.org/download/${PKG_NAME}-${PKG_VERSION}.tar.gz" -PKG_DEPENDS_HOST="" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_HOST="gcc:host" +PKG_DEPENDS_TARGET="toolchain cxxtools:host" PKG_LONGDESC="Cxxtools is a collection of general-purpose C++ classes." PKG_BUILD_FLAGS="+pic" diff --git a/packages/addons/addon-depends/dotnet-runtime-depends/curl3/package.mk b/packages/addons/addon-depends/dotnet-runtime-depends/curl3/package.mk index 7beeb3d71f..742eed76d4 100644 --- a/packages/addons/addon-depends/dotnet-runtime-depends/curl3/package.mk +++ b/packages/addons/addon-depends/dotnet-runtime-depends/curl3/package.mk @@ -5,6 +5,7 @@ PKG_NAME="curl3" PKG_URL="" +PKG_DEPENDS_UNPACK+=" curl" PKG_LONGDESC="curl for dotnet" unpack() { diff --git a/packages/addons/addon-depends/mpd-mpc/package.mk b/packages/addons/addon-depends/mpd-mpc/package.mk index 6bc74358e9..b65dd5030a 100644 --- a/packages/addons/addon-depends/mpd-mpc/package.mk +++ b/packages/addons/addon-depends/mpd-mpc/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="62373e83a8a165b2ed43967975efecd3feee530f4557d6b861dd08aa89d52b2d" PKG_LICENSE="GPL" PKG_SITE="https://www.musicpd.org" PKG_URL="https://www.musicpd.org/download/mpc/0/mpc-${PKG_VERSION}.tar.xz" -PKG_DEPENDS_TARGET="toolchain libiconv" +PKG_DEPENDS_TARGET="toolchain libiconv libmpdclient" PKG_LONGDESC="Command-line client for MPD." PKG_TOOLCHAIN="meson" diff --git a/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk b/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk index 737648d68d..e9f1989604 100644 --- a/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk +++ b/packages/addons/addon-depends/rsyslog-depends/liblognorm/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="dd779b6992de37995555e1d54caf0716a694765efc65480eed2c713105ab46fe" PKG_LICENSE="GPL" PKG_SITE="http://www.liblognorm.com" PKG_URL="https://github.com/rsyslog/liblognorm/archive/v$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain libestr" PKG_TOOLCHAIN="autotools" PKG_LONGDESC="A fast samples-based log normalization library." diff --git a/packages/addons/addon-depends/system-tools-depends/p7zip/package.mk b/packages/addons/addon-depends/system-tools-depends/p7zip/package.mk index 4db0399de1..53ef8ff7d5 100644 --- a/packages/addons/addon-depends/system-tools-depends/p7zip/package.mk +++ b/packages/addons/addon-depends/system-tools-depends/p7zip/package.mk @@ -7,6 +7,7 @@ PKG_SHA256="5eb20ac0e2944f6cb9c2d51dd6c4518941c185347d4089ea89087ffdd6e2341f" PKG_LICENSE="GPL" PKG_SITE="http://p7zip.sourceforge.net/" PKG_URL="http://downloads.sourceforge.net/project/p7zip/p7zip/${PKG_VERSION}/p7zip_${PKG_VERSION}_src_all.tar.bz2" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="p7zip is a port of 7za.exe for POSIX systems like Unix." PKG_TOOLCHAIN="manual" diff --git a/packages/addons/tools/multimedia-tools/package.mk b/packages/addons/tools/multimedia-tools/package.mk index f4d4190cc6..5fd256f238 100644 --- a/packages/addons/tools/multimedia-tools/package.mk +++ b/packages/addons/tools/multimedia-tools/package.mk @@ -21,13 +21,16 @@ PKG_ADDON_TYPE="xbmc.python.script" PKG_DEPENDS_TARGET="toolchain \ alsa-utils \ mediainfo \ - mesa-demos \ mpg123 \ opencaster \ squeezelite \ tsdecrypt \ tstools" +if [ "$TARGET_ARCH" = "x86_64" ]; then + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET mesa-demos" +fi + addon() { mkdir -p $ADDON_BUILD/$PKG_ADDON_ID/bin/ # alsamixer diff --git a/packages/addons/tools/system-tools/package.mk b/packages/addons/tools/system-tools/package.mk index 84d14418fd..b71cf8b6ee 100644 --- a/packages/addons/tools/system-tools/package.mk +++ b/packages/addons/tools/system-tools/package.mk @@ -22,7 +22,6 @@ PKG_DEPENDS_TARGET="toolchain \ diffutils \ dstat \ dtach \ - efibootmgr \ encfs \ evtest \ fdupes \ @@ -38,7 +37,6 @@ PKG_DEPENDS_TARGET="toolchain \ lm_sensors \ lshw \ mc \ - mrxvt \ mtpfs \ nmon \ p7zip \ @@ -52,6 +50,10 @@ PKG_DEPENDS_TARGET="toolchain \ usb-modeswitch \ vim" +if [ "$TARGET_ARCH" = "x86_64" ]; then + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET efibootmgr mrxvt" +fi + addon() { mkdir -p $ADDON_BUILD/$PKG_ADDON_ID/lib/ mkdir -p $ADDON_BUILD/$PKG_ADDON_ID/data/ diff --git a/packages/audio/alsa-utils/package.mk b/packages/audio/alsa-utils/package.mk index 0416893daa..c123f219af 100644 --- a/packages/audio/alsa-utils/package.mk +++ b/packages/audio/alsa-utils/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="fd9bf528922b3829a91913b89a1858c58a0b24271a7b5f529923aa9ea12fa4cf" PKG_LICENSE="GPL" PKG_SITE="http://www.alsa-project.org/" PKG_URL="ftp://ftp.alsa-project.org/pub/utils/alsa-utils-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain alsa-lib ncurses" +PKG_DEPENDS_TARGET="toolchain alsa-lib ncurses systemd" PKG_LONGDESC="This package includes the utilities for ALSA, like alsamixer, aplay, arecord, alsactl, iecset and speaker-test." PKG_CONFIGURE_OPTS_TARGET="--disable-alsaconf \ diff --git a/packages/audio/fluidsynth/package.mk b/packages/audio/fluidsynth/package.mk index 2971eeb67d..d4f1309e0c 100644 --- a/packages/audio/fluidsynth/package.mk +++ b/packages/audio/fluidsynth/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="fluidsynth" PKG_VERSION="1.1.6" @@ -7,7 +8,7 @@ PKG_SHA256="d28b47dfbf7f8e426902ae7fa2981d821fbf84f41da9e1b85be933d2d748f601" PKG_LICENSE="GPL" PKG_SITE="http://fluidsynth.org/" PKG_URL="$SOURCEFORGE_SRC/project/fluidsynth/fluidsynth-$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain glib" PKG_LONGDESC="FluidSynth renders midi music files as raw audio data, for playing or conversion." PKG_BUILD_FLAGS="+pic" diff --git a/packages/compress/bzip2/package.mk b/packages/compress/bzip2/package.mk index 610406b72e..9a128383d9 100644 --- a/packages/compress/bzip2/package.mk +++ b/packages/compress/bzip2/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="a2848f34fcd5d6cf47def00461fcb528a0484d8edef8208d6d2e2909dc61d9cd" PKG_LICENSE="GPL" PKG_SITE="http://www.bzip.org" PKG_URL="$DISTRO_SRC/$PKG_NAME-$PKG_VERSION.tar.gz" -PKG_DEPENDS_HOST="" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A high-quality bzip2 data compressor." PKG_BUILD_FLAGS="+pic +pic:host" diff --git a/packages/compress/xz/package.mk b/packages/compress/xz/package.mk index 0974846d8d..ab7d0137ce 100644 --- a/packages/compress/xz/package.mk +++ b/packages/compress/xz/package.mk @@ -9,6 +9,7 @@ PKG_LICENSE="GPL" PKG_SITE="http://tukaani.org/xz/" PKG_URL="http://tukaani.org/xz/$PKG_NAME-$PKG_VERSION.tar.bz2" PKG_DEPENDS_HOST="ccache:host" +PKG_DEPENDS_TARGET="gcc:host" PKG_LONGDESC="A free general-purpose data compression software with high compression ratio." PKG_BUILD_FLAGS="+pic" diff --git a/packages/compress/zstd/package.mk b/packages/compress/zstd/package.mk index 3ec5713722..2aee9d68cd 100644 --- a/packages/compress/zstd/package.mk +++ b/packages/compress/zstd/package.mk @@ -8,6 +8,7 @@ PKG_LICENSE="BSD/GPLv2" PKG_SITE="http://www.zstd.net" PKG_URL="https://github.com/facebook/zstd/releases/download/v${PKG_VERSION}/${PKG_NAME}-${PKG_VERSION}.tar.gz" PKG_SOURCE_DIR=$PKG_NAME-$PKG_VERSION +PKG_DEPENDS_HOST="gcc:host ninja:host" PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A fast real-time compression algorithm." diff --git a/packages/databases/sqlite/package.mk b/packages/databases/sqlite/package.mk index 66af7122b5..9bda7be10a 100644 --- a/packages/databases/sqlite/package.mk +++ b/packages/databases/sqlite/package.mk @@ -8,7 +8,8 @@ PKG_SHA256="5daa6a3fb7d1e8c767cd59c4ded8da6e4b00c61d3b466d0685e35c4dd6d7bf5d" PKG_LICENSE="PublicDomain" PKG_SITE="https://www.sqlite.org/" PKG_URL="https://www.sqlite.org/2018/$PKG_NAME-$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_HOST="gcc:host" +PKG_DEPENDS_TARGET="toolchain ncurses" PKG_LONGDESC="An Embeddable SQL Database Engine." # libsqlite3.a(sqlite3.o): requires dynamic R_X86_64_PC32 reloc against 'sqlite3_stricmp' which may overflow at runtime PKG_BUILD_FLAGS="+pic +pic:host -parallel" diff --git a/packages/debug/libunwind/package.mk b/packages/debug/libunwind/package.mk index 47698d37eb..44b91b6799 100644 --- a/packages/debug/libunwind/package.mk +++ b/packages/debug/libunwind/package.mk @@ -7,6 +7,7 @@ PKG_SHA256="3f3ecb90e28cbe53fba7a4a27ccce7aad188d3210bb1964a923a731a27a75acb" PKG_LICENSE="GPL" PKG_SITE="http://www.nongnu.org/libunwind/" PKG_URL="http://download.savannah.nongnu.org/releases/libunwind/libunwind-${PKG_VERSION}.tar.gz" +PKG_DEPENDS_TARGET="gcc:host" PKG_LONGDESC="library to determine the call-chain of a program" PKG_CONFIGURE_OPTS_TARGET="--enable-static \ diff --git a/packages/devel/autoconf/patches/autoconf-0200-performance.patch b/packages/devel/autoconf/patches/autoconf-0200-performance.patch new file mode 100644 index 0000000000..1842fe92b7 --- /dev/null +++ b/packages/devel/autoconf/patches/autoconf-0200-performance.patch @@ -0,0 +1,60 @@ +The check for solaris 'print' causes significant problems on a linux machine +with dash as /bin/sh since it triggers the execution of "print" which on some +linux systems is a perl script which is part of mailcap. Worse, this perl +script calls "which file" and if successful ignores the path file was found +in and just runs "file" without a path. Each exection causes PATH to be searched. + +Simply assuming the shell's printf function works cuts out all the fork overhead +and when parallel tasks are running, this overhead appears to be significant. + +RP +2015/11/28 +Upstream-Status: Inappropriate + +Index: autoconf-2.69/lib/m4sugar/m4sh.m4 +=================================================================== +--- autoconf-2.69.orig/lib/m4sugar/m4sh.m4 ++++ autoconf-2.69/lib/m4sugar/m4sh.m4 +@@ -1045,40 +1045,8 @@ m4_defun([_AS_ECHO_PREPARE], + [[as_nl=' + ' + export as_nl +-# Printing a long string crashes Solaris 7 /usr/bin/printf. +-as_echo='\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\' +-as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo +-as_echo=$as_echo$as_echo$as_echo$as_echo$as_echo$as_echo +-# Prefer a ksh shell builtin over an external printf program on Solaris, +-# but without wasting forks for bash or zsh. +-if test -z "$BASH_VERSION$ZSH_VERSION" \ +- && (test "X`print -r -- $as_echo`" = "X$as_echo") 2>/dev/null; then +- as_echo='print -r --' +- as_echo_n='print -rn --' +-elif (test "X`printf %s $as_echo`" = "X$as_echo") 2>/dev/null; then +- as_echo='printf %s\n' +- as_echo_n='printf %s' +-else +- if test "X`(/usr/ucb/echo -n -n $as_echo) 2>/dev/null`" = "X-n $as_echo"; then +- as_echo_body='eval /usr/ucb/echo -n "$][1$as_nl"' +- as_echo_n='/usr/ucb/echo -n' +- else +- as_echo_body='eval expr "X$][1" : "X\\(.*\\)"' +- as_echo_n_body='eval +- arg=$][1; +- case $arg in @%:@( +- *"$as_nl"*) +- expr "X$arg" : "X\\(.*\\)$as_nl"; +- arg=`expr "X$arg" : ".*$as_nl\\(.*\\)"`;; +- esac; +- expr "X$arg" : "X\\(.*\\)" | tr -d "$as_nl" +- ' +- export as_echo_n_body +- as_echo_n='sh -c $as_echo_n_body as_echo' +- fi +- export as_echo_body +- as_echo='sh -c $as_echo_body as_echo' +-fi ++as_echo='printf %s\n' ++as_echo_n='printf %s' + ]])# _AS_ECHO_PREPARE + + diff --git a/packages/devel/bison/package.mk b/packages/devel/bison/package.mk index 5c15fbc23d..886e92263c 100644 --- a/packages/devel/bison/package.mk +++ b/packages/devel/bison/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="bison" PKG_VERSION="3.0.5" @@ -7,7 +8,7 @@ PKG_SHA256="075cef2e814642e30e10e8155e93022e4a91ca38a65aa1d5467d4e969f97f338" PKG_LICENSE="GPL" PKG_SITE="http://www.gnu.org/software/bison/" PKG_URL="http://ftpmirror.gnu.org/bison/$PKG_NAME-$PKG_VERSION.tar.xz" -PKG_DEPENDS_HOST="ccache:host" +PKG_DEPENDS_HOST="ccache:host m4:host" PKG_LONGDESC="A general-purpose parser generator." PKG_CONFIGURE_OPTS_HOST="--disable-rpath --with-gnu-ld" diff --git a/packages/devel/elfutils/package.mk b/packages/devel/elfutils/package.mk index 7b67dd8e21..68a696fbf6 100644 --- a/packages/devel/elfutils/package.mk +++ b/packages/devel/elfutils/package.mk @@ -9,7 +9,7 @@ PKG_LICENSE="GPL" PKG_SITE="https://sourceware.org/elfutils/" PKG_URL="https://sourceware.org/elfutils/ftp/$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.bz2" PKG_DEPENDS_HOST="make:host zlib:host" -PKG_DEPENDS_TARGET="toolchain zlib" +PKG_DEPENDS_TARGET="toolchain zlib elfutils:host" PKG_LONGDESC="A collection of utilities to handle ELF objects." PKG_TOOLCHAIN="autotools" PKG_BUILD_FLAGS="+pic" diff --git a/packages/devel/flex/package.mk b/packages/devel/flex/package.mk index b186f9f4ca..ff61eba96b 100644 --- a/packages/devel/flex/package.mk +++ b/packages/devel/flex/package.mk @@ -8,12 +8,16 @@ PKG_SHA256="24e611ef5a4703a191012f80c1027dc9d12555183ce0ecd46f3636e587e9b8e9" PKG_LICENSE="GPL" PKG_SITE="http://flex.sourceforge.net/" PKG_URL="$SOURCEFORGE_SRC/flex/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_HOST="ccache:host" +PKG_DEPENDS_HOST="ccache:host m4:host autotools:host" +PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A tool for generating programs that perform pattern-matching on text." PKG_TOOLCHAIN="autotools" PKG_CONFIGURE_OPTS_HOST="--enable-static --disable-shared --disable-rpath --with-gnu-ld" +PKG_CONFIGURE_OPTS_TARGET="ac_cv_func_realloc_0_nonnull=yes \ + ac_cv_func_malloc_0_nonnull=yes" + post_makeinstall_host() { cat > $TOOLCHAIN/bin/lex << "EOF" #!/bin/sh diff --git a/packages/devel/glibc/package.mk b/packages/devel/glibc/package.mk index 07e93d0b45..c14d57d97a 100644 --- a/packages/devel/glibc/package.mk +++ b/packages/devel/glibc/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="f3eeb8d57e25ca9fc13c2af3dae97754f9f643bc69229546828e3a240e2af04b" PKG_LICENSE="GPL" PKG_SITE="http://www.gnu.org/software/libc/" PKG_URL="http://ftp.gnu.org/pub/gnu/glibc/$PKG_NAME-$PKG_VERSION.tar.xz" -PKG_DEPENDS_TARGET="ccache:host autotools:host autoconf:host linux:host gcc:bootstrap" +PKG_DEPENDS_TARGET="ccache:host autotools:host linux:host gcc:bootstrap" PKG_DEPENDS_INIT="glibc" PKG_LONGDESC="The Glibc package contains the main C library." PKG_BUILD_FLAGS="-gold" diff --git a/packages/devel/gmp/package.mk b/packages/devel/gmp/package.mk index fddce6009a..53eef6ca62 100644 --- a/packages/devel/gmp/package.mk +++ b/packages/devel/gmp/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="gmp" PKG_VERSION="6.1.2" @@ -7,7 +8,8 @@ PKG_SHA256="87b565e89a9a684fe4ebeeddb8399dce2599f9c9049854ca8c0dfbdea0e21912" PKG_LICENSE="LGPLv3+" PKG_SITE="http://gmplib.org/" PKG_URL="https://gmplib.org/download/gmp/$PKG_NAME-$PKG_VERSION.tar.xz" -PKG_DEPENDS_HOST="ccache:host" +PKG_DEPENDS_HOST="ccache:host m4:host" +PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A library for arbitrary precision arithmetic, operating on signed integers, rational numbers, and floating point numbers." PKG_BUILD_FLAGS="+pic:host" diff --git a/packages/devel/libcec/package.mk b/packages/devel/libcec/package.mk index ef45e2a1be..b9706ad184 100644 --- a/packages/devel/libcec/package.mk +++ b/packages/devel/libcec/package.mk @@ -32,6 +32,11 @@ else PKG_CMAKE_OPTS_TARGET="$PKG_CMAKE_OPTS_TARGET -DHAVE_AOCEC_API=0 -DHAVE_AMLOGIC_API=0" fi +# libX11 and xrandr to read the sink's EDID, used to determine the PC's HDMI physical address +if [ "$DISPLAYSERVER" = "x11" ]; then + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET libX11 libXrandr" +fi + if [ "$CEC_FRAMEWORK_SUPPORT" = "yes" ]; then PKG_PATCH_DIRS="cec-framework" PKG_CMAKE_OPTS_TARGET="$PKG_CMAKE_OPTS_TARGET -DHAVE_LINUX_API=1" @@ -49,6 +54,9 @@ pre_configure_target() { } post_makeinstall_target() { + # Remove the Python3 demo - useless for us + rm -f $INSTALL/usr/bin/pyCecClient + PYTHON_DIR=$INSTALL/usr/lib/$PKG_PYTHON_VERSION if [ -d $PYTHON_DIR/dist-packages ]; then mv $PYTHON_DIR/dist-packages $PYTHON_DIR/site-packages diff --git a/packages/devel/libffi/package.mk b/packages/devel/libffi/package.mk index b4e83eb32b..0cb80e05dd 100644 --- a/packages/devel/libffi/package.mk +++ b/packages/devel/libffi/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="libffi" PKG_VERSION="3.2.1" @@ -7,6 +8,7 @@ PKG_SHA256="d06ebb8e1d9a22d19e38d63fdb83954253f39bedc5d46232a05645685722ca37" PKG_LICENSE="GPL" PKG_SITE="http://sourceware.org/$PKG_NAME/" PKG_URL="ftp://sourceware.org/pub/$PKG_NAME/$PKG_NAME-$PKG_VERSION.tar.gz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="Foreign Function Interface Library." PKG_TOOLCHAIN="autotools" diff --git a/packages/devel/ncurses/package.mk b/packages/devel/ncurses/package.mk index c04d685099..e5797340f3 100644 --- a/packages/devel/ncurses/package.mk +++ b/packages/devel/ncurses/package.mk @@ -7,6 +7,7 @@ PKG_SHA256="08b07c3e792961f300829512c283d5fefc0b1c421a57b76922c3d13303ed677d" PKG_LICENSE="MIT" PKG_SITE="http://www.gnu.org/software/ncurses/" PKG_URL="http://invisible-mirror.net/archives/ncurses/current/ncurses-$PKG_VERSION.tgz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain zlib ncurses:host" PKG_LONGDESC="A library is a free software emulation of curses in System V Release 4.0, and more." # causes some segmentation fault's (dialog) when compiled with gcc's link time optimization. diff --git a/packages/devel/parallel/package.mk b/packages/devel/parallel/package.mk new file mode 100644 index 0000000000..9b62df3999 --- /dev/null +++ b/packages/devel/parallel/package.mk @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +PKG_NAME="parallel" +PKG_VERSION="20190122" +PKG_SHA256="ae735f201a8ceeff2ace48ff779bda9d19846e629bcc02ea7c8768a42394190c" +PKG_LICENSE="GPLv3" +PKG_SITE="https://www.gnu.org/software/parallel/" +PKG_URL="http://ftpmirror.gnu.org/parallel/$PKG_NAME-$PKG_VERSION.tar.bz2" +PKG_DEPENDS_HOST="" +PKG_LONGDESC="GNU parallel is a shell tool for executing jobs in parallel using one or more computers." diff --git a/packages/devel/pcre/package.mk b/packages/devel/pcre/package.mk index 0518214abf..314832467b 100644 --- a/packages/devel/pcre/package.mk +++ b/packages/devel/pcre/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="2cd04b7c887808be030254e8d77de11d3fe9d4505c39d4b15d2664ffe8bf9301" PKG_LICENSE="OSS" PKG_SITE="http://www.pcre.org/" PKG_URL="https://ftp.pcre.org/pub/pcre/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_HOST="" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="A set of functions that implement regular expression pattern matching." PKG_TOOLCHAIN="configure" diff --git a/packages/lang/Python2/package.mk b/packages/lang/Python2/package.mk index a5e17dae8e..edfe51e6b0 100644 --- a/packages/lang/Python2/package.mk +++ b/packages/lang/Python2/package.mk @@ -10,7 +10,7 @@ PKG_LICENSE="OSS" PKG_SITE="http://www.python.org/" PKG_URL="http://www.python.org/ftp/python/$PKG_VERSION/${PKG_NAME::-1}-$PKG_VERSION.tar.xz" PKG_DEPENDS_HOST="zlib:host bzip2:host sqlite:host" -PKG_DEPENDS_TARGET="toolchain sqlite expat zlib bzip2 openssl libffi Python2:host" +PKG_DEPENDS_TARGET="toolchain sqlite expat zlib bzip2 openssl libffi Python2:host ncurses readline" PKG_LONGDESC="Python2 is an interpreted object-oriented programming language." PKG_TOOLCHAIN="autotools" diff --git a/packages/lang/gcc/package.mk b/packages/lang/gcc/package.mk index f899aeb2b1..a7d1d87c74 100644 --- a/packages/lang/gcc/package.mk +++ b/packages/lang/gcc/package.mk @@ -11,6 +11,7 @@ PKG_URL="http://ftpmirror.gnu.org/gcc/$PKG_NAME-$PKG_VERSION/$PKG_NAME-$PKG_VERS PKG_DEPENDS_BOOTSTRAP="ccache:host autoconf:host binutils:host gmp:host mpfr:host mpc:host" PKG_DEPENDS_TARGET="gcc:host" PKG_DEPENDS_HOST="ccache:host autoconf:host binutils:host gmp:host mpfr:host mpc:host glibc" +PKG_DEPENDS_INIT="toolchain" PKG_LONGDESC="This package contains the GNU Compiler Collection." GCC_COMMON_CONFIGURE_OPTS="--target=$TARGET_NAME \ diff --git a/packages/linux/package.mk b/packages/linux/package.mk index a363af2c0c..f9804d98c2 100644 --- a/packages/linux/package.mk +++ b/packages/linux/package.mk @@ -5,8 +5,8 @@ PKG_NAME="linux" PKG_LICENSE="GPL" PKG_SITE="http://www.kernel.org" -PKG_DEPENDS_HOST="ccache:host" -PKG_DEPENDS_TARGET="toolchain cpio:host kmod:host pciutils xz:host wireless-regdb keyutils $KERNEL_EXTRA_DEPENDS_TARGET" +PKG_DEPENDS_HOST="ccache:host openssl:host" +PKG_DEPENDS_TARGET="toolchain cpio:host kmod:host xz:host wireless-regdb keyutils $KERNEL_EXTRA_DEPENDS_TARGET" PKG_DEPENDS_INIT="toolchain" PKG_NEED_UNPACK="$LINUX_DEPENDS" PKG_LONGDESC="This package contains a precompiled kernel image and the modules." @@ -57,7 +57,7 @@ if [ "$PKG_BUILD_PERF" != "no" ] && grep -q ^CONFIG_PERF_EVENTS= $PKG_KERNEL_CFG fi if [ "$TARGET_ARCH" = "x86_64" ]; then - PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET intel-ucode:host kernel-firmware elfutils:host" + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET intel-ucode:host kernel-firmware elfutils:host pciutils" fi if [ "$BUILD_ANDROID_BOOTIMG" = "yes" ]; then diff --git a/packages/mediacenter/JsonSchemaBuilder/package.mk b/packages/mediacenter/JsonSchemaBuilder/package.mk index b61af9fcfc..91fefc90ec 100644 --- a/packages/mediacenter/JsonSchemaBuilder/package.mk +++ b/packages/mediacenter/JsonSchemaBuilder/package.mk @@ -1,11 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="JsonSchemaBuilder" PKG_VERSION="0" PKG_LICENSE="GPL" PKG_SITE="http://www.kodi.tv" -PKG_URL="" +PKG_DEPENDS_HOST="toolchain" +PKG_DEPENDS_UNPACK="kodi" PKG_NEED_UNPACK="$(get_pkg_directory $MEDIACENTER)" PKG_LONGDESC="kodi-platform:" diff --git a/packages/mediacenter/TexturePacker/package.mk b/packages/mediacenter/TexturePacker/package.mk index 8faa689907..ed8ee075be 100644 --- a/packages/mediacenter/TexturePacker/package.mk +++ b/packages/mediacenter/TexturePacker/package.mk @@ -1,12 +1,13 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="TexturePacker" PKG_VERSION="0" PKG_LICENSE="GPL" PKG_SITE="http://www.kodi.tv" -PKG_URL="" PKG_DEPENDS_HOST="lzo:host libpng:host libjpeg-turbo:host giflib:host" +PKG_DEPENDS_UNPACK="kodi" PKG_NEED_UNPACK="$(get_pkg_directory $MEDIACENTER)" PKG_LONGDESC="kodi-platform:" diff --git a/packages/mediacenter/kodi/package.mk b/packages/mediacenter/kodi/package.mk index 27452eddbb..a49cd50b57 100644 --- a/packages/mediacenter/kodi/package.mk +++ b/packages/mediacenter/kodi/package.mk @@ -5,7 +5,7 @@ PKG_NAME="kodi" PKG_LICENSE="GPL" PKG_SITE="http://www.kodi.tv" -PKG_DEPENDS_TARGET="toolchain JsonSchemaBuilder:host TexturePacker:host Python2 zlib systemd pciutils lzo pcre swig:host libass curl fontconfig fribidi tinyxml libjpeg-turbo freetype libcdio taglib libxml2 libxslt rapidjson sqlite ffmpeg crossguid giflib libdvdnav libhdhomerun libfmt lirc libfstrcmp flatbuffers:host flatbuffers" +PKG_DEPENDS_TARGET="toolchain JsonSchemaBuilder:host TexturePacker:host Python2 zlib systemd lzo pcre swig:host libass curl fontconfig fribidi tinyxml libjpeg-turbo freetype libcdio taglib libxml2 libxslt rapidjson sqlite ffmpeg crossguid giflib libdvdnav libhdhomerun libfmt lirc libfstrcmp flatbuffers:host flatbuffers" PKG_LONGDESC="A free and open source cross-platform media player." PKG_PATCH_DIRS="$KODI_VENDOR" @@ -39,6 +39,10 @@ configure_package() { get_graphicdrivers + if [ "$TARGET_ARCH" = "x86_64" ]; then + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET pciutils" + fi + PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET dbus" if [ "$DISPLAYSERVER" = "x11" ]; then diff --git a/packages/multimedia/dav1d/package.mk b/packages/multimedia/dav1d/package.mk index 275aca4d13..ab20356a49 100644 --- a/packages/multimedia/dav1d/package.mk +++ b/packages/multimedia/dav1d/package.mk @@ -7,5 +7,9 @@ PKG_SHA256="3662184809fa29af39c920877cedec3724dc5c3cf71d94bec424f2a7ee4825cf" PKG_LICENSE="BSD" PKG_SITE="http://www.jbkempf.com/blog/post/2018/Introducing-dav1d" PKG_URL="https://code.videolan.org/videolan/dav1d/-/archive/${PKG_VERSION}/dav1d-${PKG_VERSION}.tar.bz2" -PKG_DEPENDS_TARGET="toolchain nasm:host" +PKG_DEPENDS_TARGET="toolchain" PKG_LONGDESC="dav1d is an AV1 decoder :)" + +if [ "$TARGET_ARCH" = "x86_64" ]; then + PKG_DEPENDS_TARGET+=" nasm:host" +fi diff --git a/packages/print/freetype/package.mk b/packages/print/freetype/package.mk index 4188f2c974..23759c30d0 100644 --- a/packages/print/freetype/package.mk +++ b/packages/print/freetype/package.mk @@ -8,6 +8,7 @@ PKG_SHA256="e20a6e1400798fd5e3d831dd821b61c35b1f9a6465d6b18a53a9df4cf441acf0" PKG_LICENSE="GPL" PKG_SITE="http://www.freetype.org" PKG_URL="http://download.savannah.gnu.org/releases/freetype/$PKG_NAME-$PKG_VERSION.tar.bz2" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain zlib libpng" PKG_LONGDESC="The FreeType engine is a free and portable TrueType font rendering engine." PKG_TOOLCHAIN="configure" diff --git a/packages/python/devel/Mako/package.mk b/packages/python/devel/Mako/package.mk index 45007cf755..41217f6f55 100644 --- a/packages/python/devel/Mako/package.mk +++ b/packages/python/devel/Mako/package.mk @@ -13,5 +13,5 @@ PKG_LONGDESC="Mako is a super-fast templating language that borrows the best ide PKG_TOOLCHAIN="manual" makeinstall_host() { - python3 setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python3 setup.py install --prefix=$TOOLCHAIN } diff --git a/packages/python/devel/MarkupSafe/package.mk b/packages/python/devel/MarkupSafe/package.mk index 378038122d..20c090e602 100644 --- a/packages/python/devel/MarkupSafe/package.mk +++ b/packages/python/devel/MarkupSafe/package.mk @@ -13,5 +13,5 @@ PKG_LONGDESC="MarkupSafe implements a XML/HTML/XHTML Markup safe string for Pyth PKG_TOOLCHAIN="manual" makeinstall_host() { - python3 setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python3 setup.py install --prefix=$TOOLCHAIN } diff --git a/packages/python/devel/distutilscross/package.mk b/packages/python/devel/distutilscross/package.mk index 367e35a7c0..4c767be5ee 100644 --- a/packages/python/devel/distutilscross/package.mk +++ b/packages/python/devel/distutilscross/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="distutilscross" PKG_VERSION="0.1" @@ -12,5 +13,5 @@ PKG_LONGDESC="distutilscross enhances distutils to support Cross Compile of Pyth PKG_TOOLCHAIN="manual" makeinstall_host() { - python setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python setup.py install --prefix=$TOOLCHAIN } diff --git a/packages/python/devel/meson/package.mk b/packages/python/devel/meson/package.mk index c6b4ced8e5..022ef607ae 100644 --- a/packages/python/devel/meson/package.mk +++ b/packages/python/devel/meson/package.mk @@ -16,7 +16,7 @@ make_host() { } makeinstall_host() { - python3 setup.py install --prefix=$TOOLCHAIN --skip-build + exec_thread_safe python3 setup.py install --prefix=$TOOLCHAIN --skip-build # Avoid using full path to python3 that may exceed 128 byte limit. # Instead use PATH as we know our toolchain is first. diff --git a/packages/python/devel/pathlib/package.mk b/packages/python/devel/pathlib/package.mk index 30c9beaa60..61f8e55a6f 100644 --- a/packages/python/devel/pathlib/package.mk +++ b/packages/python/devel/pathlib/package.mk @@ -12,5 +12,5 @@ PKG_LONGDESC="This module offers a set of classes featuring all the common opera PKG_TOOLCHAIN="manual" makeinstall_host() { - python3 setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python3 setup.py install --prefix=$TOOLCHAIN } diff --git a/packages/python/devel/setuptools/package.mk b/packages/python/devel/setuptools/package.mk index 8a21964cfa..531e6d7238 100644 --- a/packages/python/devel/setuptools/package.mk +++ b/packages/python/devel/setuptools/package.mk @@ -18,6 +18,6 @@ make_host() { } makeinstall_host() { - python2 setup.py install --prefix=$TOOLCHAIN - python3 setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python2 setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python3 setup.py install --prefix=$TOOLCHAIN } diff --git a/packages/security/nspr/package.mk b/packages/security/nspr/package.mk index 7d16ac8df0..58f4f2200f 100644 --- a/packages/security/nspr/package.mk +++ b/packages/security/nspr/package.mk @@ -1,12 +1,14 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2012 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="nspr" PKG_VERSION="4.19" PKG_LICENSE="Mozilla Public License" PKG_SITE="http://www.linuxfromscratch.org/blfs/view/svn/general/nspr.html" PKG_DEPENDS_HOST="ccache:host" -PKG_DEPENDS_TARGET="toolchain nss:host" +PKG_DEPENDS_TARGET="toolchain nss:host nspr:host" +PKG_DEPENDS_UNPACK="nss" PKG_LONGDESC="Netscape Portable Runtime (NSPR) provides a platform-neutral API for system level and libc like functions" PKG_TOOLCHAIN="configure" PKG_BUILD_FLAGS="-parallel" diff --git a/packages/sysutils/busybox/package.mk b/packages/sysutils/busybox/package.mk index c39c4c2ef1..247ddfc4fc 100644 --- a/packages/sysutils/busybox/package.mk +++ b/packages/sysutils/busybox/package.mk @@ -8,8 +8,8 @@ PKG_SHA256="9553da068c0a30b1b8b72479908c1ba58672e2be7b535363a88de5e0f7bc04ce" PKG_LICENSE="GPL" PKG_SITE="http://www.busybox.net" PKG_URL="http://busybox.net/downloads/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_HOST="" -PKG_DEPENDS_TARGET="toolchain busybox:host hdparm dosfstools e2fsprogs zip unzip pciutils usbutils parted procps-ng gptfdisk libtirpc" +PKG_DEPENDS_HOST="gcc:host" +PKG_DEPENDS_TARGET="toolchain busybox:host hdparm dosfstools e2fsprogs zip unzip usbutils parted procps-ng gptfdisk libtirpc" PKG_DEPENDS_INIT="toolchain libtirpc" PKG_LONGDESC="BusyBox combines tiny versions of many common UNIX utilities into a single small executable." # busybox fails to build with GOLD support enabled with binutils-2.25 @@ -25,6 +25,10 @@ if [ "$NFS_SUPPORT" = yes ]; then PKG_DEPENDS_TARGET="$PKG_DEPENDS_TARGET rpcbind" fi +if [ "$TARGET_ARCH" = "x86_64" ]; then + PKG_DEPENDS_TARGET+=" pciutils" +fi + pre_build_target() { PKG_MAKE_OPTS_TARGET="ARCH=$TARGET_ARCH \ HOSTCC=$HOST_CC \ diff --git a/packages/sysutils/dosfstools/package.mk b/packages/sysutils/dosfstools/package.mk index 6f8c72b92b..65bc096aaa 100644 --- a/packages/sysutils/dosfstools/package.mk +++ b/packages/sysutils/dosfstools/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="dosfstools" PKG_VERSION="3.0.28" @@ -7,6 +8,7 @@ PKG_SHA256="ee95913044ecf2719b63ea11212917649709a6e53209a72d622135aaa8517ee2" PKG_LICENSE="GPLv3" PKG_SITE="https://github.com/dosfstools/dosfstools" PKG_URL="https://github.com/dosfstools/dosfstools/releases/download/v$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.xz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_DEPENDS_INIT="toolchain dosfstools" PKG_LONGDESC="dosfstools contains utilities for making and checking MS-DOS FAT filesystems." diff --git a/packages/sysutils/e2fsprogs/package.mk b/packages/sysutils/e2fsprogs/package.mk index 65ba2b8c66..410ada8963 100644 --- a/packages/sysutils/e2fsprogs/package.mk +++ b/packages/sysutils/e2fsprogs/package.mk @@ -8,6 +8,7 @@ PKG_SHA256="926f8e8de1ffba55d791f21b71334e8a32b5227257ad370f2bf7e4396629e97f" PKG_LICENSE="GPL" PKG_SITE="http://e2fsprogs.sourceforge.net/" PKG_URL="https://www.kernel.org/pub/linux/kernel/people/tytso/$PKG_NAME/v$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.xz" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain" PKG_DEPENDS_INIT="toolchain" PKG_LONGDESC="The filesystem utilities for the EXT2 filesystem, including e2fsck, mke2fs, dumpe2fs, fsck, and others." diff --git a/packages/sysutils/gptfdisk/package.mk b/packages/sysutils/gptfdisk/package.mk index 2668ea675a..5e59cb072b 100644 --- a/packages/sysutils/gptfdisk/package.mk +++ b/packages/sysutils/gptfdisk/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="b663391a6876f19a3cd901d862423a16e2b5ceaa2f4a3b9bb681e64b9c7ba78d" PKG_LICENSE="GPL" PKG_SITE="http://www.rodsbooks.com/gdisk/" PKG_URL="https://downloads.sourceforge.net/project/$PKG_NAME/$PKG_NAME/$PKG_VERSION/$PKG_NAME-$PKG_VERSION.tar.gz" -PKG_DEPENDS_TARGET="toolchain popt" +PKG_DEPENDS_TARGET="toolchain popt crossguid" PKG_LONGDESC="GPT text-mode partitioning tools" make_target() { diff --git a/packages/sysutils/lirc/package.mk b/packages/sysutils/lirc/package.mk index 5358c9eab1..c273a591e4 100644 --- a/packages/sysutils/lirc/package.mk +++ b/packages/sysutils/lirc/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="lirc" PKG_VERSION="0.10.0" @@ -13,7 +14,9 @@ PKG_TOOLCHAIN="autotools" PKG_PYTHON_WANTED=Python2 -PKG_CONFIGURE_OPTS_TARGET="--enable-devinput \ +PKG_CONFIGURE_OPTS_TARGET="ac_cv_header_alsa_asoundlib_h=no \ + ac_cv_lib_asound_snd_async_del_handler=no \ + --enable-devinput \ --with-gnu-ld \ --without-x \ --runstatedir=/run" diff --git a/packages/sysutils/util-linux/package.mk b/packages/sysutils/util-linux/package.mk index 07e07e8ea7..27a18b83b3 100644 --- a/packages/sysutils/util-linux/package.mk +++ b/packages/sysutils/util-linux/package.mk @@ -7,7 +7,7 @@ PKG_VERSION="2.32.1" PKG_SHA256="86e6707a379c7ff5489c218cfaf1e3464b0b95acf7817db0bc5f179e356a67b2" PKG_LICENSE="GPL" PKG_URL="http://www.kernel.org/pub/linux/utils/util-linux/v${PKG_VERSION%.*}/$PKG_NAME-$PKG_VERSION.tar.xz" -PKG_DEPENDS_HOST="" +PKG_DEPENDS_HOST="gcc:host pkg-config:host" PKG_DEPENDS_TARGET="toolchain" PKG_DEPENDS_INIT="toolchain" PKG_LONGDESC="A large variety of low-level system utilities that are necessary for a Linux system to function." diff --git a/packages/sysutils/v4l-utils/package.mk b/packages/sysutils/v4l-utils/package.mk index 6e37014745..84495dfe29 100644 --- a/packages/sysutils/v4l-utils/package.mk +++ b/packages/sysutils/v4l-utils/package.mk @@ -10,7 +10,7 @@ PKG_SHA256="e6b962c4b1253cf852c31da13fd6b5bb7cbe5aa9e182881aec55123bae680692" PKG_LICENSE="GPL" PKG_SITE="http://linuxtv.org/" PKG_URL="http://linuxtv.org/downloads/v4l-utils/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain alsa-lib systemd" PKG_LONGDESC="Linux V4L2 and DVB API utilities and v4l libraries (libv4l)." PKG_CONFIGURE_OPTS_TARGET="--without-jpeg \ diff --git a/packages/tools/aml-dtbtools/package.mk b/packages/tools/aml-dtbtools/package.mk index 8e21b9bbeb..93a0d28f72 100644 --- a/packages/tools/aml-dtbtools/package.mk +++ b/packages/tools/aml-dtbtools/package.mk @@ -7,6 +7,7 @@ PKG_SHA256="8bcaa83fcc9e85c9c04930e7411447d96a97da0809c5ecd9af91c8b554133c41" PKG_LICENSE="free" PKG_SITE="https://github.com/Wilhansen/aml-dtbtools" PKG_URL="https://github.com/Wilhansen/aml-dtbtools/archive/${PKG_VERSION}.tar.gz" +PKG_DEPENDS_HOST="gcc:host" PKG_LONGDESC="AML DTB Tools" PKG_MAKE_OPTS_HOST="dtbTool" diff --git a/packages/tools/dtc/package.mk b/packages/tools/dtc/package.mk index e943d3e790..c5a005a774 100644 --- a/packages/tools/dtc/package.mk +++ b/packages/tools/dtc/package.mk @@ -8,7 +8,7 @@ PKG_LICENSE="GPL" PKG_SITE="https://git.kernel.org/pub/scm/utils/dtc/dtc.git/" PKG_URL="https://git.kernel.org/pub/scm/utils/dtc/dtc.git/snapshot/$PKG_VERSION.tar.gz" PKG_DEPENDS_HOST="Python2:host swig:host" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain dtc:host" PKG_LONGDESC="The Device Tree Compiler" PKG_MAKE_OPTS_HOST="dtc libfdt" @@ -22,7 +22,7 @@ makeinstall_host() { post_makeinstall_host() { python ./pylibfdt/setup.py build_ext --inplace - python ./pylibfdt/setup.py install --prefix=$TOOLCHAIN + exec_thread_safe python ./pylibfdt/setup.py install --prefix=$TOOLCHAIN touch $TOOLCHAIN/lib/$PKG_PYTHON_VERSION/site-packages/pylibfdt/__init__.py } diff --git a/packages/tools/mkbootimg/package.mk b/packages/tools/mkbootimg/package.mk index 0b9d83a380..adc1d3acd6 100644 --- a/packages/tools/mkbootimg/package.mk +++ b/packages/tools/mkbootimg/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="mkbootimg" PKG_VERSION="6668fc2" @@ -7,7 +8,7 @@ PKG_SHA256="d84870e055414d638a3e7eb4b7a3ebf415899841218f24cb3647d06ecf6ddb17" PKG_LICENSE="GPL" PKG_SITE="https://android.googlesource.com/platform/system/core/+/master/mkbootimg/" PKG_URL="https://github.com/codesnake/mkbootimg/archive/$PKG_VERSION.tar.gz" -PKG_DEPENDS_HOST="" +PKG_DEPENDS_HOST="gcc:host" PKG_LONGDESC="mkbootimg: Creates kernel boot images for Android" makeinstall_host() { diff --git a/packages/tools/mtools/package.mk b/packages/tools/mtools/package.mk index 099488fbc7..e0dd42e0e8 100644 --- a/packages/tools/mtools/package.mk +++ b/packages/tools/mtools/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="mtools" PKG_VERSION="4.0.18" @@ -7,6 +8,6 @@ PKG_SHA256="59e9cf80885399c4f229e5d87e49c0c2bfeec044e1386d59fcd0b0aead6b2f85" PKG_LICENSE="GPL" PKG_SITE="http://www.gnu.org/software/mtools/" PKG_URL="http://ftpmirror.gnu.org/$PKG_NAME/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_HOST="" +PKG_DEPENDS_HOST="gcc:host" PKG_LONGDESC="mtools: A collection of utilities to access MS-DOS disks" PKG_TOOLCHAIN="autotools" diff --git a/packages/virtual/image/package.mk b/packages/virtual/image/package.mk new file mode 100644 index 0000000000..12ecfaacfe --- /dev/null +++ b/packages/virtual/image/package.mk @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +PKG_NAME="image" +PKG_LICENSE="GPL" +PKG_SITE="https://libreelec.tv" +PKG_DEPENDS_TARGET="toolchain squashfs:host dosfstools:host fakeroot:host kmod:host mtools:host populatefs:host libc gcc linux linux-drivers linux-firmware ${BOOTLOADER} busybox util-linux corefonts network misc-packages debug" +PKG_SECTION="virtual" +PKG_LONGDESC="Root package used to build and create complete image" + +# Graphic support +[ ! "$DISPLAYSERVER" = "no" ] && PKG_DEPENDS_TARGET+=" $DISPLAYSERVER" + +# Multimedia support +[ ! "$MEDIACENTER" = "no" ] && PKG_DEPENDS_TARGET+=" mediacenter" + +# Sound support +[ "$ALSA_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" alsa" + +# Automounter support +[ "$UDEVIL" = "yes" ] && PKG_DEPENDS_TARGET+=" udevil" + +# EXFAT support +[ "$EXFAT" = "yes" ] && PKG_DEPENDS_TARGET+=" fuse-exfat" + +# NTFS 3G support +[ "$NTFS3G" = "yes" ] && PKG_DEPENDS_TARGET+=" ntfs-3g_ntfsprogs" + +# Remote support +[ "$REMOTE_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" remote" + +# Virtual image creation support +[ "$PROJECT" = "Generic" ] && PKG_DEPENDS_TARGET+=" virtual" + +# Installer support +[ "$INSTALLER_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" installer" + +# Devtools... (not for Release) +[ "$TESTING" = "yes" ] && PKG_DEPENDS_TARGET+=" testing" + +# OEM packages +[ "$OEM_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" oem" + +true diff --git a/packages/virtual/initramfs/package.mk b/packages/virtual/initramfs/package.mk index ef17f196ce..b278804659 100644 --- a/packages/virtual/initramfs/package.mk +++ b/packages/virtual/initramfs/package.mk @@ -22,14 +22,14 @@ fi post_install() { ( cd $BUILD/initramfs if [ "$TARGET_ARCH" = "x86_64" ]; then - ln -sf /usr/lib $BUILD/initramfs/lib64 + ln -sfn /usr/lib $BUILD/initramfs/lib64 mkdir -p $BUILD/initramfs/usr - ln -sf /usr/lib $BUILD/initramfs/usr/lib64 + ln -sfn /usr/lib $BUILD/initramfs/usr/lib64 fi - ln -sf /usr/lib $BUILD/initramfs/lib - ln -sf /usr/bin $BUILD/initramfs/bin - ln -sf /usr/sbin $BUILD/initramfs/sbin + ln -sfn /usr/lib $BUILD/initramfs/lib + ln -sfn /usr/bin $BUILD/initramfs/bin + ln -sfn /usr/sbin $BUILD/initramfs/sbin mkdir -p $BUILD/image/ fakeroot -- sh -c \ diff --git a/packages/x11/app/xkbcomp/package.mk b/packages/x11/app/xkbcomp/package.mk index fdee2edcb6..cefb1e3750 100644 --- a/packages/x11/app/xkbcomp/package.mk +++ b/packages/x11/app/xkbcomp/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="xkbcomp" PKG_VERSION="1.4.1" @@ -7,7 +8,7 @@ PKG_SHA256="748dc4cf58ac95684106bd9cf163ac6ab7de9a236faec02a6f4d4006d63a5736" PKG_LICENSE="OSS" PKG_SITE="http://www.X.org" PKG_URL="http://xorg.freedesktop.org/archive/individual/app/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain util-macros libX11" +PKG_DEPENDS_TARGET="toolchain util-macros libX11 libxkbfile" PKG_LONGDESC="The xkbcomp keymap compiler converts a description of an XKB keymap into one of several output formats." PKG_CONFIGURE_OPTS_TARGET="--with-xkb-config-root=$XORG_PATH_XKB" diff --git a/packages/x11/driver/xf86-input-evdev/package.mk b/packages/x11/driver/xf86-input-evdev/package.mk index e83dd8198c..c015cfcd95 100644 --- a/packages/x11/driver/xf86-input-evdev/package.mk +++ b/packages/x11/driver/xf86-input-evdev/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="9edaa6205baf6d2922cc4db3d8e54a7e7773b5f733b0ae90f6be7725f983b70d" PKG_LICENSE="OSS" PKG_SITE="http://www.X.org" PKG_URL="http://xorg.freedesktop.org/archive/individual/driver/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain util-macros libevdev mtdev systemd" +PKG_DEPENDS_TARGET="toolchain xorg-server util-macros libevdev mtdev systemd" PKG_LONGDESC="Evdev is an Xorg input driver for Linux's generic event devices." PKG_TOOLCHAIN="autotools" diff --git a/packages/x11/driver/xf86-input-synaptics/package.mk b/packages/x11/driver/xf86-input-synaptics/package.mk index abbdf178aa..c867909778 100644 --- a/packages/x11/driver/xf86-input-synaptics/package.mk +++ b/packages/x11/driver/xf86-input-synaptics/package.mk @@ -7,7 +7,7 @@ PKG_SHA256="afba3289d7a40217a19d90db98ce181772f9ca6d77e1898727b0afcf02073b5a" PKG_LICENSE="GPL" PKG_SITE="http://lists.freedesktop.org/mailman/listinfo/xorg" PKG_URL="http://xorg.freedesktop.org/archive/individual/driver/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain libXi" +PKG_DEPENDS_TARGET="toolchain xorg-server libXi libXext libevdev" PKG_LONGDESC="Synaptics touchpad driver for X.Org." PKG_TOOLCHAIN="autotools" diff --git a/packages/x11/font/font-bitstream-type1/package.mk b/packages/x11/font/font-bitstream-type1/package.mk index 276cd585e2..a0e4695506 100644 --- a/packages/x11/font/font-bitstream-type1/package.mk +++ b/packages/x11/font/font-bitstream-type1/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="font-bitstream-type1" PKG_VERSION="1.0.3" @@ -7,7 +8,7 @@ PKG_SHA256="c6ea0569adad2c577f140328dc3302e729cb1b1ea90cd0025caf380625f8a688" PKG_LICENSE="OSS" PKG_SITE="http://www.X.org" PKG_URL="http://xorg.freedesktop.org/archive/individual/font/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain util-macros" +PKG_DEPENDS_TARGET="toolchain util-macros font-xfree86-type1" PKG_LONGDESC="Bitstream font family." PKG_CONFIGURE_OPTS_TARGET="--with-fontrootdir=/usr/share/fonts" diff --git a/packages/x11/lib/libXi/package.mk b/packages/x11/lib/libXi/package.mk index 8f50cc5f9c..ac1707c94e 100644 --- a/packages/x11/lib/libXi/package.mk +++ b/packages/x11/lib/libXi/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="c2e6b8ff84f9448386c1b5510a5cf5a16d788f76db018194dacdc200180faf45" PKG_LICENSE="OSS" PKG_SITE="http://www.x.org/" PKG_URL="http://xorg.freedesktop.org/archive/individual/lib/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain util-macros libX11 libXfixes" +PKG_DEPENDS_TARGET="toolchain util-macros libX11 libXfixes libXext" PKG_LONGDESC="LibXi provides an X Window System client interface to the XINPUT extension to the X protocol." PKG_BUILD_FLAGS="+pic" diff --git a/packages/x11/lib/libxshmfence/package.mk b/packages/x11/lib/libxshmfence/package.mk index daa5868c7b..63d48f66aa 100644 --- a/packages/x11/lib/libxshmfence/package.mk +++ b/packages/x11/lib/libxshmfence/package.mk @@ -8,7 +8,7 @@ PKG_SHA256="b884300d26a14961a076fbebc762a39831cb75f92bed5ccf9836345b459220c7" PKG_LICENSE="OSS" PKG_SITE="http://www.X.org" PKG_URL="http://xorg.freedesktop.org/archive/individual/lib/$PKG_NAME-$PKG_VERSION.tar.bz2" -PKG_DEPENDS_TARGET="toolchain util-macros" +PKG_DEPENDS_TARGET="toolchain util-macros xorgproto" PKG_LONGDESC="libxshmfence is the Shared memory 'SyncFence' synchronization primitive." PKG_TOOLCHAIN="autotools" PKG_BUILD_FLAGS="+pic" diff --git a/packages/x11/lib/pixman/package.mk b/packages/x11/lib/pixman/package.mk index 65868f22f9..631a570444 100644 --- a/packages/x11/lib/pixman/package.mk +++ b/packages/x11/lib/pixman/package.mk @@ -1,5 +1,6 @@ # SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) PKG_NAME="pixman" PKG_VERSION="0.34.0" @@ -7,6 +8,7 @@ PKG_SHA256="39ba3438f3d17c464b0cb8be006dacbca0ab5aee97ebde69fec7ecdbf85794a0" PKG_LICENSE="OSS" PKG_SITE="http://www.x.org/" PKG_URL="http://xorg.freedesktop.org/archive/individual/lib/$PKG_NAME-$PKG_VERSION.tar.bz2" +PKG_DEPENDS_HOST="gcc:host" PKG_DEPENDS_TARGET="toolchain util-macros" PKG_LONGDESC="Pixman is a generic library for manipulating pixel regions, contains low-level pixel manipulation routines." diff --git a/projects/Amlogic_Legacy/packages/autoscript-amlogic/package.mk b/projects/Amlogic_Legacy/packages/autoscript-amlogic/package.mk index b35cfacb70..ce08e3feb4 100644 --- a/projects/Amlogic_Legacy/packages/autoscript-amlogic/package.mk +++ b/projects/Amlogic_Legacy/packages/autoscript-amlogic/package.mk @@ -4,7 +4,7 @@ PKG_NAME="autoscript-amlogic" PKG_VERSION="" PKG_LICENSE="GPL" -PKG_DEPENDS_TARGET="toolchain" +PKG_DEPENDS_TARGET="toolchain u-boot-tools-aml:host" PKG_LONGDESC="Autoscript package for Amlogic devices" PKG_TOOLCHAIN="manual" diff --git a/scripts/autoreconf b/scripts/autoreconf index 1ad473fe38..26cc2ca1bc 100755 --- a/scripts/autoreconf +++ b/scripts/autoreconf @@ -12,11 +12,24 @@ if [ ! -f "${PKG_BUILD}/configure.in" \ fi if [ ! -f "${PKG_BUILD}/.autoreconf-done" ]; then - touch "${PKG_BUILD}/NEWS" "${PKG_BUILD}/AUTHORS" "${PKG_BUILD}/ChangeLog" - mkdir -p "${PKG_BUILD}/m4" + PARENT_PKG="${2}" - build_msg "CLR_AUTORECONF" "AUTORECONF" "${1}" "indent" + # lock package during autoreconf otherwise it is racy, eg. glib:host/glib:target building concurrently + pkg_lock "${PKG_NAME}" "reconf" "${PARENT_PKG}" - do_autoreconf "${PKG_BUILD}" - touch "${PKG_BUILD}/.autoreconf-done" + if [ ! -f "${PKG_BUILD}/.autoreconf-done" ]; then + pkg_lock_status "ACTIVE" "${PKG_NAME}" "reconf" + + touch "${PKG_BUILD}/NEWS" "${PKG_BUILD}/AUTHORS" "${PKG_BUILD}/ChangeLog" + mkdir -p "${PKG_BUILD}/m4" + + build_msg "CLR_AUTORECONF" "AUTORECONF" "${PKG_NAME}" "indent" + + do_autoreconf "${PKG_BUILD}" + touch "${PKG_BUILD}/.autoreconf-done" + + pkg_lock_status "UNLOCK" "${PKG_NAME}" "reconf" "configured" + else + pkg_lock_status "UNLOCK" "${PKG_NAME}" "reconf" "already configured" + fi fi diff --git a/scripts/build b/scripts/build index 1d240b2a12..3317a1dfe3 100755 --- a/scripts/build +++ b/scripts/build @@ -7,7 +7,7 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name[:]" + die "usage: $0 package_name[:] [parent_pkg]" fi if [ "$1" = "--all" ]; then @@ -32,10 +32,13 @@ if [ "${1//:/}" != "${1}" ]; then PACKAGE_NAME="${1%:*}" TARGET="${1#*:}" else - PACKAGE_NAME=$1 + PACKAGE_NAME="${1}" TARGET= fi TARGET="${TARGET:-target}" +PARENT_PKG="${2:-${PKG_NAME}:${TARGET}}" + +pkg_lock "${PACKAGE_NAME}:${TARGET}" "build" "${PARENT_PKG}" mkdir -p $STAMPS/$PACKAGE_NAME STAMP=$STAMPS/$PACKAGE_NAME/build_$TARGET @@ -46,15 +49,21 @@ if [ -f $STAMP ]; then rm -f $STAMP elif [ ! "$BUILD_WITH_DEBUG" = "$STAMP_BUILD_WITH_DEBUG" ]; then rm -f $STAMP - elif [ "$1" = "u-boot" -a ! "$UBOOT_SYSTEM" = "$STAMP_UBOOT_SYSTEM" ]; then + elif [ "${PKG_NAME}" = "u-boot" -a ! "$UBOOT_SYSTEM" = "$STAMP_UBOOT_SYSTEM" ]; then rm -f $STAMP else # stamp matched: already built, do nothing + pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "already built" exit 0 fi fi -$SCRIPTS/unpack $PACKAGE_NAME +if [ -n "${PKG_DEPENDS_UNPACK}" ]; then + for p in ${PKG_DEPENDS_UNPACK}; do + $SCRIPTS/unpack "${p}" "${PARENT_PKG}" + done +fi +$SCRIPTS/unpack "${PACKAGE_NAME}" "${PARENT_PKG}" # build dependencies, only when PKG_DEPENDS_? is filled unset _pkg_depends @@ -65,16 +74,9 @@ case "$TARGET" in "bootstrap") _pkg_depends="$PKG_DEPENDS_BOOTSTRAP";; esac for p in $_pkg_depends; do - $SCRIPTS/build $p + $SCRIPTS/build "${p}" "${PARENT_PKG}" done -# build this package -if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then - build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})") [DEBUG]" "indent" -else - build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})")" "indent" -fi - # virtual packages are not built as they only contain dependencies, so dont go further here if [ "$PKG_SECTION" = "virtual" ]; then PKG_DEEPHASH=$(calculate_stamp) @@ -82,9 +84,17 @@ if [ "$PKG_SECTION" = "virtual" ]; then echo "STAMP_$i=\"${!i}\"" >> $STAMP done + pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "built" exit 0 fi +# build this package +if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then + build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})") [DEBUG]" "indent" +else + build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})")" "indent" +fi + setup_toolchain $TARGET # configure install directory @@ -201,9 +211,11 @@ build_msg "CLR_TOOLCHAIN" "TOOLCHAIN" "${PKG_TOOLCHAIN}${_auto_toolchain}" # make autoreconf if [ "$PKG_TOOLCHAIN" = "autotools" ]; then - $SCRIPTS/autoreconf $PACKAGE_NAME + $SCRIPTS/autoreconf "${PACKAGE_NAME}" "${PARENT_PKG}" fi +pkg_lock_status "ACTIVE" "${PKG_NAME}:${TARGET}" "build" + # include build template and build pkg_call_exists pre_build_$TARGET && pkg_call pre_build_$TARGET @@ -361,6 +373,14 @@ fi pkg_call_exists post_make_$TARGET && pkg_call post_make_$TARGET +# make install writes directly to sysroot which then needs fixing up - this is racy +if listcontains "configure:target cmake-make:target autotools:target make:target" "${PKG_TOOLCHAIN}:${TARGET}"; then + acquire_exclusive_lock "${PKG_NAME}:${TARGET}" "build" + PKG_NEED_UNLOCK=yes +else + PKG_NEED_UNLOCK=no +fi + # make install pkg_call_exists pre_makeinstall_$TARGET && pkg_call pre_makeinstall_$TARGET @@ -385,7 +405,7 @@ else # make based builds "configure:target"|"cmake-make:target"|"autotools:target"|"make:target") - $MAKEINSTALL $PKG_MAKEINSTALL_OPTS_TARGET + make install DESTDIR=$SYSROOT_PREFIX -j1 $PKG_MAKEINSTALL_OPTS_TARGET make install DESTDIR=$INSTALL $PKG_MAKEINSTALL_OPTS_TARGET ;; "configure:host"|"cmake-make:host"|"autotools:host"|"make:host") @@ -402,6 +422,15 @@ fi pkg_call_exists post_makeinstall_$TARGET && pkg_call post_makeinstall_$TARGET +# This is the racy part - one or more processes may be creating, modifying or deleting +# $SYSROOT_PREFIX/usr/lib/*.la files while another (eg. this) process is rewriting the same +# files as it fixes up the libdir prefix. Eugh. +for i in $(find $SYSROOT_PREFIX/usr/lib/ -name "*.la" 2>/dev/null); do + sed -e "s:\(['= ]\)/usr:\\1$SYSROOT_PREFIX/usr:g" -i $i +done + +[ "${PKG_NEED_UNLOCK}" = "yes" ] && release_exclusive_lock "${PKG_NAME}:${TARGET}" "build" || true + if [ "$TARGET" = "target" -o "$TARGET" = "init" ]; then if [ -d $INSTALL ]; then rm -rf $INSTALL/{usr/,}include @@ -447,14 +476,12 @@ fi cd $ROOT -for i in $(find $SYSROOT_PREFIX/usr/lib/ -name "*.la" 2>/dev/null); do - sed -e "s:\(['= ]\)/usr:\\1$SYSROOT_PREFIX/usr:g" -i $i -done - PKG_DEEPHASH=$(calculate_stamp) for i in PKG_NAME PKG_DEEPHASH BUILD_WITH_DEBUG; do echo "STAMP_$i=\"${!i}\"" >> $STAMP done -if [ "$1" = "u-boot" ]; then +if [ "${PKG_NAME}" = "u-boot" ]; then echo "STAMP_UBOOT_SYSTEM=\"${UBOOT_SYSTEM}\"" >> $STAMP fi + +pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "built" diff --git a/scripts/create_addon b/scripts/create_addon index 64588696d7..2388dab05f 100755 --- a/scripts/create_addon +++ b/scripts/create_addon @@ -1,16 +1,15 @@ #!/bin/bash -# SPDX-License-Identifier: GPL-2.0-or-later -# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) -# Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv) +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) . config/options "" +. config/multithread -# usage usage() { - cat - >&2 <&2 < ./script/create_addon all + > ./script/create_addon_mt all build audio encoders and decoders, only - > ./script/create_addon audioencoder.* audiodecoder.* + > ./script/create_addon_mt audioencoder.* audiodecoder.* build all, but not binary - > ./script/create_addon all -binary - -EOUSAGE - exit ${1:0} + > ./script/create_addon_mt all -binary +EOF + exit ${1:-0} } -# functions -function find_addons() { - local _paths="" - local _filter="." +# Get list of addon packages +get_addons() { + local paths filter + local pkgpath exited + local count=0 validpkg + case $1 in - binary) _paths="$ROOT/packages/mediacenter/kodi-binary-addons";; - official) _paths="$ROOT/packages/addons";; - all) _paths="$ROOT/packages $ROOT/projects/*/packages";; - *) _paths="$ROOT/packages $ROOT/projects/*/packages"; - _filter="^$1$";; + binary) paths="^$ROOT/packages/mediacenter/kodi-binary-addons/";; + official) paths="^$ROOT/packages/addons/";; + all) paths="^$ROOT/packages/|^$ROOT/projects/.*/packages/";; + *) paths="^$ROOT/packages/|^$ROOT/projects/.*/packages/"; filter="$1";; esac - local _addons=$( - find $_paths -name 'package.mk' \ - `# select packages with PKG_IS_ADDON (can yes, no or unset at this moment)` \ - | xargs grep -l 'PKG_IS_ADDON' \ - `# extract package name from path` \ - | sed 's|^.*/\([^/]*\)/package.mk$|\1|g' \ - `# filter package list against the given filter` \ - | grep -e "$_filter" \ - `# make entries unique` \ - | sort -u \ - `# select packages with PKG_IS_ADDON=yes (slow, but is a short list, now)` \ - | xargs -n1 -I{} /bin/bash -c '. ./config/options {} &>/dev/null; [ "$PKG_IS_ADDON" = "yes" ] && echo $PKG_NAME' - ) + exit() { exited=1; } - # check if anything is found - local _count=$(wc -w <<< $_addons) - if [ $_count -eq 0 ]; then - # handle embedded addons here. Should only build when they are explictly specified in the addon list - ( . ./config/options "$1" &>/dev/null - [ "$PKG_IS_ADDON" != "embedded" -a "$PKG_IS_ADDON" != "yes" ] && exit 1 - echo $PKG_NAME - ) - - # abort when nothing found and not embedded - if [ $? -ne 0 ]; then - echo "$(print_color CLR_ERROR "ERROR: '$1' matches nothing...")" >&$SILENT_OUT - echo "for more informations type: ./scripts/create_addon --help" >&$SILENT_OUT - die + for pkgpath in $(cat "${_CACHE_PACKAGE_LOCAL}" "${_CACHE_PACKAGE_GLOBAL}" | grep -E "${paths}"); do + if [ -n "${filter}" ]; then + [[ ${pkgpath} =~ ^.*/${filter}@?+?@ ]] || continue fi - fi - echo $_addons -} + exited=0 + source_package "${pkgpath%%@*}/package.mk" &>/dev/null + [ ${exited} -eq 1 ] && continue -pack_addon() { - scripts/install_addon $PKG_NAME $PKG_ADDON_ID || exit - - if [ "$2" != "-test" ] ; then - ADDON_INSTALL_DIR="$TARGET_IMG/$ADDONS/$ADDON_VERSION/${DEVICE:-$PROJECT}/$TARGET_ARCH/$PKG_ADDON_ID" - ADDONVER="$(xmlstarlet sel -t -v "/addon/@version" $ADDON_BUILD/$PKG_ADDON_ID/addon.xml)" - - if [ -f $ADDON_INSTALL_DIR/$PKG_ADDON_ID-$ADDONVER.zip ]; then - if [ "$ADDON_OVERWRITE" = "yes" ]; then - rm $ADDON_INSTALL_DIR/$PKG_ADDON_ID-$ADDONVER.zip - else - build_msg "CLR_WARNING" "*** WARNING: ${PKG_ADDON_ID}-${ADDONVER}.zip already exists. Not overwrittng it. ***" - return 0 + validpkg="no" + # Should only build embedded addons when they are explictly specified in the addon list + if [ "${PKG_IS_ADDON}" = "embedded" ]; then + if [ -n "${filter}" ]; then + verify_addon && validpkg="yes" fi - fi - cd $ADDON_BUILD - build_msg "CLR_INFO" "*** compressing addon ${PKG_ADDON_ID} ... ***" - $TOOLCHAIN/bin/7za a -l -mx9 -bsp0 -bso0 -tzip $PKG_ADDON_ID-$ADDONVER.zip $PKG_ADDON_ID - cd - &>/dev/null - - mkdir -p $ADDON_INSTALL_DIR - cp $ADDON_BUILD/$PKG_ADDON_ID-$ADDONVER.zip $ADDON_INSTALL_DIR - if [ -f $ADDON_BUILD/$PKG_ADDON_ID/changelog.txt ]; then - cp $ADDON_BUILD/$PKG_ADDON_ID/changelog.txt $ADDON_INSTALL_DIR/changelog-$ADDONVER.txt - fi - if [ -f $ADDON_BUILD/$PKG_ADDON_ID/resources/icon.png ]; then - mkdir -p $ADDON_INSTALL_DIR/resources - cp $ADDON_BUILD/$PKG_ADDON_ID/resources/icon.png $ADDON_INSTALL_DIR/resources/icon.png + elif [ "${PKG_IS_ADDON}" = "yes" ]; then + verify_addon && validpkg="yes" fi - # workaround for kodi pvr addons - if [ -f $ADDON_BUILD/$PKG_ADDON_ID/icon.png ]; then - cp $ADDON_BUILD/$PKG_ADDON_ID/icon.png $ADDON_INSTALL_DIR/icon.png + if [ "${validpkg}" = "yes" ]; then + echo "${PKG_NAME}" + count=$((count + 1)) fi + done - if [ -f $ADDON_BUILD/$PKG_ADDON_ID/resources/fanart.png ]; then - mkdir -p $ADDON_INSTALL_DIR/resources - cp $ADDON_BUILD/$PKG_ADDON_ID/resources/fanart.png $ADDON_INSTALL_DIR/resources/fanart.png - fi - for f in $ADDON_BUILD/$PKG_ADDON_ID/resources/screenshot-*.{jpg,png}; do - if [ -f "$f" ]; then - mkdir -p $ADDON_INSTALL_DIR/resources - cp $f $ADDON_INSTALL_DIR/resources - fi - done + unset -f exit - # Jenkins add-on build - if [ "$ADDON_JENKINS" = "yes" ]; then - ADDON_JENKINS_DIR="$TARGET_IMG/jenkins" - ADDON_JENKINS_ADDON_NAME="$ADDON_VERSION-${DEVICE:-$PROJECT}-$TARGET_ARCH-$PKG_ADDON_ID-$ADDONVER" - mkdir -p "$ADDON_JENKINS_DIR" - cd $ADDON_INSTALL_DIR - $TOOLCHAIN/bin/7za a -l -mx0 -bsp0 -bso0 -tzip $ADDON_JENKINS_DIR/$ADDON_JENKINS_ADDON_NAME.zip $PKG_ADDON_ID-$ADDONVER.zip resources/ - ( cd $ADDON_JENKINS_DIR - sha256sum $ADDON_JENKINS_ADDON_NAME.zip > $ADDON_JENKINS_ADDON_NAME.zip.sha256 - ) - build_msg "CLR_INFO" "*** creating ${ADDON_JENKINS_ADDON_NAME}.zip for Jenkins complete ***" - else - build_msg "CLR_INFO" "*** creating ${PKG_ADDON_ID} complete ***" - fi + if [ ${count} -eq 0 -a -n "${filter}" ]; then + echo "$(print_color CLR_ERROR "ERROR: no addons matched for filter ${filter}")" >&2 + echo "For more information type: ./scripts/create_addon_mt --help" >&2 + die fi } -not_supported_arch() { - build_msg "CLR_WARNING" "*** SKIP: ${PKG_ADDON_ID} '${TARGET_ARCH}' not supported ***" - exit 0 -} +# Return 0 if package is a suitable addon, 1 otherwise +verify_addon() { + [ "${PKG_ADDON_TYPE}" = "xbmc.broken" ] && return 1 -not_supported_device() { - build_msg "CLR_WARNING" "*** SKIP: ${PKG_ADDON_ID}: '${DEVICE:-${PROJECT}}' not supported ***" - exit 0 -} - -# build addon function -build_addon() { - # addon build - . config/options $1 - - # check support - if [ -n "$PKG_ARCH" ]; then - listcontains "$PKG_ARCH" "!$TARGET_ARCH" && not_supported_arch - listcontains "$PKG_ARCH" "$TARGET_ARCH" || listcontains "$PKG_ARCH" "any" || not_supported_arch + if [ -n "${PKG_ARCH}" ]; then + listcontains "${PKG_ARCH}" "!${TARGET_ARCH}" && return 1 + listcontains "${PKG_ARCH}" "${TARGET_ARCH}" || listcontains "${PKG_ARCH}" "any" || return 1 fi - if [ -n "$PKG_ADDON_PROJECTS" ]; then - [ "${DEVICE}" = "RPi" ] && _DEVICE="RPi1" || _DEVICE="${DEVICE}" + if [ -n "${PKG_ADDON_PROJECTS}" ]; then + [ "${DEVICE}" = "RPi" ] && _DEVICE="RPi1" || _DEVICE="${DEVICE}" - if listcontains "$PKG_ADDON_PROJECTS" "!${_DEVICE:-$PROJECT}" || - listcontains "$PKG_ADDON_PROJECTS" "!${PROJECT}"; then - not_supported_device + if listcontains "${PKG_ADDON_PROJECTS}" "!${_DEVICE:-$PROJECT}" || + listcontains "${PKG_ADDON_PROJECTS}" "!${PROJECT}"; then + return 1 fi - if ! listcontains "$PKG_ADDON_PROJECTS" "${_DEVICE:-$PROJECT}" && - ! listcontains "$PKG_ADDON_PROJECTS" "${PROJECT}" && - ! listcontains "$PKG_ADDON_PROJECTS" "any"; then - not_supported_device + if ! listcontains "${PKG_ADDON_PROJECTS}" "${_DEVICE:-$PROJECT}" && + ! listcontains "${PKG_ADDON_PROJECTS}" "${PROJECT}" && + ! listcontains "${PKG_ADDON_PROJECTS}" "any"; then + return 1 fi fi - # build addon - $SCRIPTS/build $1 - - # cleanup old install path - rm -rf $ADDON_BUILD - - # install addon parts - if pkg_call_exists addon; then - pkg_call addon - else - install_binary_addon $PKG_ADDON_ID - fi - - # HACK for packages that provide multiple addons like screensavers.rsxs - # addon's addon() in package.mk should take care for exporting - # MULTI_ADDONS="addon.boo1 addon.boo2 addon.boo3" - if [ -n "$MULTI_ADDONS" ] ; then - for _ADDON in $MULTI_ADDONS ; do - PKG_ADDON_ID=$_ADDON - pack_addon - done - else - pack_addon - fi + return 0 } # need parameter @@ -232,98 +125,60 @@ if [ $# -eq 0 ]; then usage 1 fi +# check environment and configure toolchains +${SCRIPTS}/checkdeps + +(setup_toolchain host) +setup_toolchain target + # collect list of addons for building -addons="" -addons_drop="" -show_only="false" -write_logs="no" -log_path="$BUILD/logs" -summary_file="/dev/null" -export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE)) +addons= +addons_drop= +show_only="no" # read addons from parameter list while [ $# -gt 0 ]; do case $1 in --help) usage 0;; - --show-only) show_only="true";; - --write-logs=*) write_logs="${1:13}";; - --log-path=*) log_path="${1:11}";; + --show-only) show_only="yes";; --*) usage 1;; - -*) addons_drop+=" $(find_addons ${1:1})";; - *) addons+=" $(find_addons $1)";; + -*) addons_drop+=" $(get_addons ${1:1})";; + *) addons+=" $(get_addons $1)";; esac shift done -# check log parameter -case "$write_logs" in - no) log_path="" - remove_success_logs="false";; - yes) remove_success_logs="false";; - errors) remove_success_logs="true";; - *) usage 1 -esac -if [ -n "$log_path" ]; then - mkdir -p "$log_path" - summary_file="$log_path/addon-summary.log" - rm -f $summary_file -fi - -# check environment and create toolchain -$SCRIPTS/checkdeps -setup_toolchain target - -# build addons, by calling function build_addon with one addon, after another -# (do not abort on build failure) -addons_failed="" -set +e -_count='' -for addon in $(tr " " "\n" <<< $addons | sort -u); do - # no build, when addon is in drop list / should not build - if listcontains "$addons_drop" "$addon"; then - continue - fi - - # show-only: print name and continue with next addon - if [ "$show_only" = "true" ]; then - echo $addon - continue - fi - - # define log file - log_file=/dev/null - if [ -n "$log_path" ]; then - log_file="$log_path/$addon.log" - fi - - # build package - echo "$(print_color CLR_BUILD "CREATE ADDON $addon") (${DEVICE:-$PROJECT}/$TARGET_ARCH)" >&$SILENT_OUT - _count+='x' - ( build_addon $addon ) \ - 2>&1 | tee $log_file - if [ ${PIPESTATUS[0]} != 0 ]; then - addons_failed+="$addon " - echo "$(print_color CLR_ERROR "ADDON FAILED $addon")" >&$SILENT_OUT - echo "failed: ${addon}" >> $summary_file - else - if [ "$remove_success_logs" = "true" ]; then - rm -f $log_file - fi - echo "success: ${addon}" >> $summary_file - fi +# Build a new list containing only those addons we want to build +wanted_addons= +for addon in $(echo ${addons} | tr ' ' '\n' | sort -u); do + listcontains "${addons_drop}" "${addon}" || wanted_addons+=" ${addon}" done -# show-only has no summary, can exit here -if [ "$show_only" = "true" ]; then +if [ "${show_only}" = "yes" ]; then + for addon in ${wanted_addons}; do + echo ${addon} + done exit 0 fi -# print summary -if [ "$_count" != 'x' ]; then - if [ -z "$addons_failed" ]; then - echo "$(print_color CLR_INFO "ALL ADDONS BUILDS SUCCESSFUL")" - exit 0 - else - die "$(print_color CLR_ERROR "FAILED ADDONS: $addons_failed")" - fi +# Build all addons at once using a single plan +if MTADDONBUILD=yes start_multithread_build "${wanted_addons:1}"; then + echo + echo "$(print_color CLR_INFO "ALL ADDONS BUILT SUCCESSFULLY")" + exit 0 +elif [ -f "${THREAD_CONTROL}/addons.failed" ]; then + echo >&2 + print_color CLR_ERROR "FAILED ADDONS:\n" >&2 + while read -r addon logfile; do + if [ -n "${addon}" ]; then + if [ -n "${logfile}" ]; then + echo " $(print_color CLR_ERROR "${addon}"): ${logfile}" >&2 + else + echo " $(print_color CLR_ERROR "${addon}")" >&2 + fi + fi + done < "${THREAD_CONTROL}/addons.failed" + die +else + die "$(print_color CLR_ERROR "UNKNOWN BUILD FAILURE OR INABILITY TO GENERATE PLAN")" fi diff --git a/scripts/create_addon_st b/scripts/create_addon_st new file mode 100755 index 0000000000..f2e75d6505 --- /dev/null +++ b/scripts/create_addon_st @@ -0,0 +1,266 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2018-present Team LibreELEC (https://libreelec.tv) + +. config/options "" + +# usage +usage() { + cat - >&2 < ./script/create_addon all + + build audio encoders and decoders, only + > ./script/create_addon audioencoder.* audiodecoder.* + + build all, but not binary + > ./script/create_addon all -binary + +EOUSAGE + exit ${1:0} +} + +# functions +function find_addons() { + local _paths="" + local _filter="." + case $1 in + binary) _paths="$ROOT/packages/mediacenter/kodi-binary-addons";; + official) _paths="$ROOT/packages/addons";; + all) _paths="$ROOT/packages $ROOT/projects/*/packages";; + *) _paths="$ROOT/packages $ROOT/projects/*/packages"; + _filter="^$1$";; + esac + + local _addons=$( + find $_paths -name 'package.mk' \ + `# select packages with PKG_IS_ADDON (can yes, no or unset at this moment)` \ + | xargs grep -l 'PKG_IS_ADDON' \ + `# extract package name from path` \ + | sed 's|^.*/\([^/]*\)/package.mk$|\1|g' \ + `# filter package list against the given filter` \ + | grep -e "$_filter" \ + `# make entries unique` \ + | sort -u \ + `# select packages with PKG_IS_ADDON=yes (slow, but is a short list, now)` \ + | xargs -n1 -I{} /bin/bash -c '. ./config/options {} &>/dev/null; [ "$PKG_IS_ADDON" = "yes" ] && echo $PKG_NAME' + ) + + # check if anything is found + local _count=$(wc -w <<< $_addons) + if [ $_count -eq 0 ]; then + # handle embedded addons here. Should only build when they are explictly specified in the addon list + ( . ./config/options "$1" &>/dev/null + [ "$PKG_IS_ADDON" != "embedded" -a "$PKG_IS_ADDON" != "yes" ] && exit 1 + echo $PKG_NAME + ) + + # abort when nothing found and not embedded + if [ $? -ne 0 ]; then + echo "$(print_color CLR_ERROR "ERROR: '$1' matches nothing...")" >&$SILENT_OUT + echo "for more informations type: ./scripts/create_addon --help" >&$SILENT_OUT + die + fi + fi + + echo $_addons +} + +not_supported_arch() { + build_msg "CLR_WARNING" "*** SKIP: ${PKG_ADDON_ID} '${TARGET_ARCH}' not supported ***" + exit 0 +} + +not_supported_device() { + build_msg "CLR_WARNING" "*** SKIP: ${PKG_ADDON_ID}: '${DEVICE:-${PROJECT}}' not supported ***" + exit 0 +} + +# build addon function +build_addon() { + # addon build + . config/options $1 + + # check support + if [ -n "$PKG_ARCH" ]; then + listcontains "$PKG_ARCH" "!$TARGET_ARCH" && not_supported_arch + listcontains "$PKG_ARCH" "$TARGET_ARCH" || listcontains "$PKG_ARCH" "any" || not_supported_arch + fi + + if [ -n "$PKG_ADDON_PROJECTS" ]; then + [ "${DEVICE}" = "RPi" ] && _DEVICE="RPi1" || _DEVICE="${DEVICE}" + + if listcontains "$PKG_ADDON_PROJECTS" "!${_DEVICE:-$PROJECT}" || + listcontains "$PKG_ADDON_PROJECTS" "!${PROJECT}"; then + not_supported_device + fi + + if ! listcontains "$PKG_ADDON_PROJECTS" "${_DEVICE:-$PROJECT}" && + ! listcontains "$PKG_ADDON_PROJECTS" "${PROJECT}" && + ! listcontains "$PKG_ADDON_PROJECTS" "any"; then + not_supported_device + fi + fi + + # build addon + $SCRIPTS/build $1 + + # cleanup old install path + rm -rf $ADDON_BUILD + + # install addon parts + if pkg_call_exists addon; then + pkg_call addon + else + install_binary_addon $PKG_ADDON_ID + fi + + # HACK for packages that provide multiple addons like screensavers.rsxs + # addon's addon() in package.mk should take care for exporting + # MULTI_ADDONS="addon.boo1 addon.boo2 addon.boo3" + if [ -n "$MULTI_ADDONS" ] ; then + for _ADDON in $MULTI_ADDONS ; do + PKG_ADDON_ID=$_ADDON + ${SCRIPTS}/install_addon $PKG_NAME $PKG_ADDON_ID + done + else + ${SCRIPTS}/install_addon $PKG_NAME $PKG_ADDON_ID + fi +} + +# need parameter +if [ $# -eq 0 ]; then + usage 1 +fi + +# collect list of addons for building +addons="" +addons_drop="" +show_only="false" +write_logs="no" +log_path="$BUILD/logs" +summary_file="/dev/null" +export BUILD_INDENT=$((${BUILD_INDENT:-1}+$BUILD_INDENT_SIZE)) + +# read addons from parameter list +while [ $# -gt 0 ]; do + case $1 in + --help) usage 0;; + --show-only) show_only="true";; + --write-logs=*) write_logs="${1:13}";; + --log-path=*) log_path="${1:11}";; + --*) usage 1;; + -*) addons_drop+=" $(find_addons ${1:1})";; + *) addons+=" $(find_addons $1)";; + esac + shift +done + +# check log parameter +case "$write_logs" in + no) log_path="" + remove_success_logs="false";; + yes) remove_success_logs="false";; + errors) remove_success_logs="true";; + *) usage 1 +esac +if [ -n "$log_path" ]; then + mkdir -p "$log_path" + summary_file="$log_path/addon-summary.log" + rm -f $summary_file +fi + +# check environment and create toolchain +$SCRIPTS/checkdeps +setup_toolchain target + +# build addons, by calling function build_addon with one addon, after another +# (do not abort on build failure) +addons_failed="" +set +e +_count='' +for addon in $(tr " " "\n" <<< $addons | sort -u); do + # no build, when addon is in drop list / should not build + if listcontains "$addons_drop" "$addon"; then + continue + fi + + # show-only: print name and continue with next addon + if [ "$show_only" = "true" ]; then + echo $addon + continue + fi + + # define log file + log_file=/dev/null + if [ -n "$log_path" ]; then + log_file="$log_path/$addon.log" + fi + + # build package + echo "$(print_color CLR_BUILD "CREATE ADDON $addon") (${DEVICE:-$PROJECT}/$TARGET_ARCH)" >&$SILENT_OUT + _count+='x' + ( build_addon $addon ) \ + 2>&1 | tee $log_file + if [ ${PIPESTATUS[0]} != 0 ]; then + addons_failed+="$addon " + echo "$(print_color CLR_ERROR "ADDON FAILED $addon")" >&$SILENT_OUT + echo "failed: ${addon}" >> $summary_file + else + if [ "$remove_success_logs" = "true" ]; then + rm -f $log_file + fi + echo "success: ${addon}" >> $summary_file + fi +done + +# show-only has no summary, can exit here +if [ "$show_only" = "true" ]; then + exit 0 +fi + +# print summary +if [ "$_count" != 'x' ]; then + if [ -z "$addons_failed" ]; then + echo "$(print_color CLR_INFO "ALL ADDONS BUILDS SUCCESSFUL")" + exit 0 + else + die "$(print_color CLR_ERROR "FAILED ADDONS: $addons_failed")" + fi +fi diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py new file mode 100755 index 0000000000..73a5d175d5 --- /dev/null +++ b/scripts/genbuildplan.py @@ -0,0 +1,388 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +from __future__ import print_function +import sys, os, codecs, json, argparse, re + +ROOT_PKG = "__root__" + +class LibreELEC_Package: + def __init__(self, name, section): + self.name = name + self.section = section + self.deps = {"bootstrap": [], + "init": [], + "host": [], + "target": []} + self.wants = [] + self.wantedby = [] + + def __repr__(self): + s = "%-9s: %s" % ("name", self.name) + s = "%s\n%-9s: %s" % (s, "section", self.section) + + for t in self.deps: + s = "%s\n%-9s: %s" % (s, t, self.deps[t]) + + s = "%s\n%-9s: %s" % (s, "NEEDS", self.wants) + s = "%s\n%-9s: %s" % (s, "WANTED BY", self.wantedby) + + return s + + def addDependencies(self, target, packages): + for d in " ".join(packages.split()).split(): + self.deps[target].append(d) + name = d.split(":")[0] + if name not in self.wants and name != self.name: + self.wants.append(name) + + def delDependency(self, target, package): + if package in self.deps[target]: + self.deps[target].remove(package) + name = package.split(":")[0] + if name in self.wants: + self.wants.remove(name) + + def addReference(self, package): + name = package.split(":")[0] + if name not in self.wantedby: + self.wantedby.append(name) + + def delReference(self, package): + name = package.split(":")[0] + if name in self.wantedby: + self.wantedby.remove(name) + + def isReferenced(self): + return False if self.wants == [] else True + + def isWanted(self): + return False if self.wantedby == [] else True + + def references(self, package): + return package in self.wants + +# Reference material: +# https://www.electricmonk.nl/docs/dependency_resolving_algorithm/dependency_resolving_algorithm.html +class Node: + def __init__(self, name, target, section): + self.name = name + self.target = target + self.section = section + self.fqname = "%s:%s" % (name, target) + self.edges = [] + + def appendEdges(self, node): + # Add the node itself... + if node not in self.edges: + self.edges.append(node) + # as well as its edges + for e in node.edges: + if e not in self.edges: + self.edges.append(e) + + # Return True if the dependencies of the specified node are met by this node + def satisfies(self, node): + for e in node.edges: + if e not in self.edges: + return False + return True + + def __repr__(self): + s = "%-9s: %s" % ("name", self.name) + s = "%s\n%-9s: %s" % (s, "target", self.target) + s = "%s\n%-9s: %s" % (s, "fqname", self.fqname) + s = "%s\n%-9s: %s" % (s, "common", self.commonName()) + s = "%s\n%-9s: %s" % (s, "section", self.section) + + for e in self.edges: + s = "%s\nEDGE: %s" % (s, e.fqname) + + return s + + def commonName(self): + return self.name if self.target == "target" else "%s:%s" % (self.name, self.target) + + def addEdge(self, node): + self.edges.append(node) + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +# Read a JSON list of all possible packages from stdin +def loadPackages(): + jdata = json.loads("[%s]" % sys.stdin.read().replace('\n','')[:-1]) + + map = {} + + # Load "global" packages first + for pkg in jdata: + if pkg["hierarchy"] == "global": + map[pkg["name"]] = initPackage(pkg) + + # Then the "local" packages, as these will replace any matching "global" packages + for pkg in jdata: + if pkg["hierarchy"] == "local": + map[pkg["name"]] = initPackage(pkg) + + return map + +# Create a fully formed LibreELEC_Package object +def initPackage(package): + pkg = LibreELEC_Package(package["name"], package["section"]) + + for target in ["bootstrap", "init", "host", "target"]: + pkg.addDependencies(target, package[target]) + + return pkg + +# Split name:target into components +def split_package(name): + parts = name.split(":") + pn = parts[0] + pt = parts[1] if len(parts) != 1 else "target" + return (pn, pt) + +# Return a list of packages of the specified type +def get_packages_by_target(target, list): + newlist = [] + + for p in list: + (pn, pt) = split_package(p) + if target in ["target", "init"] and pt in ["target", "init"]: + newlist.append(p) + elif target in ["bootstrap", "host"] and pt in ["bootstrap", "host"]: + newlist.append(p) + + return newlist + +# For the specified node iterate over the list of scheduled nodes and return the first +# position where we could possibly build this node (ie. all dependencies satisfied). +def findbuildpos(node, list): + + # Keep a running total of all dependencies as we progress through the list + alldeps = Node("", "", "") + + candidate = None + for n in list: + alldeps.appendEdges(n) + if alldeps.satisfies(node): + if len(n.edges) > len(node.edges): + if candidate == None: + candidate = n + break + candidate = n + + return list.index(candidate) + 1 if candidate else -1 + +# Resolve dependencies for a node +def dep_resolve(node, resolved, unresolved, noreorder): + unresolved.append(node) + + for edge in node.edges: + if edge not in resolved: + if edge in unresolved: + raise Exception('Circular reference detected: %s -> %s\nRemove %s from %s package.mk::PKG_DEPENDS_%s' % \ + (node.fqname, edge.commonName(), edge.commonName(), node.name, node.target.upper())) + dep_resolve(edge, resolved, unresolved, noreorder) + + if node not in resolved: + pos = -1 if noreorder else findbuildpos(node, resolved) + if pos != -1: + resolved.insert(pos, node) + else: + resolved.append(node) + + unresolved.remove(node) + +# Return a list of build steps for the trigger packages +def get_build_steps(args, nodes, trigger_pkgs, built_pkgs): + resolved = [] + unresolved = [] + + # When building the image the :target packages must be installed. + # + # However, if we are not building the image then only build the packages + # and don't install them as it's likely we will be building discrete add-ons + # which are installed outside of the image. + # + install = True if "image" in args.build else False + + for pkgname in [x for x in trigger_pkgs if x]: + if pkgname.find(":") == -1: + pkgname = "%s:target" % pkgname + + if pkgname in nodes: + dep_resolve(nodes[pkgname], resolved, unresolved, args.no_reorder) + + # Abort if any references remain unresolved + if unresolved != []: + eprint("The following dependencies have not been resolved:") + for dep in unresolved: + eprint(" %s" % dep) + raise("Unresolved references") + + # Output list of resolved dependencies + for pkg in resolved: + if pkg.fqname not in built_pkgs: + built_pkgs.append(pkg.fqname) + task = "build" if pkg.fqname.endswith(":host") or not install else "install" + yield(task, pkg.fqname) + +# Reduce the complete list of packages to a map of those packages that will +# be needed for the build. +def processPackages(args, packages, build): + # Add dummy package to ensure build/install dependencies are not culled + pkg = { + "name": ROOT_PKG, + "section": "virtual", + "hierarchy": "global", + "bootstrap": "", + "init": "", + "host": " ".join(get_packages_by_target("host", build)), + "target": " ".join(get_packages_by_target("target", build)) + } + + packages[pkg["name"]] = initPackage(pkg) + + # Resolve reverse references that we can use to ignore unreferenced packages + for pkgname in packages: + for opkgname in packages: + opkg = packages[opkgname] + if opkg.references(pkgname): + if pkgname in packages: + packages[pkgname].addReference(opkgname) + + # Identify unused packages + while True: + changed = False + for pkgname in packages: + pkg = packages[pkgname] + if pkg.isWanted(): + for opkgname in pkg.wantedby: + if opkgname != ROOT_PKG: + if not packages[opkgname].isWanted(): + pkg.delReference(opkgname) + changed = True + if not changed: + break + + # Create a new map of "needed" packages + needed_map = {} + for pkgname in packages: + pkg = packages[pkgname] + if pkg.isWanted() or pkgname == ROOT_PKG: + needed_map[pkgname] = pkg + + # Validate package dependency references + for pkgname in needed_map: + pkg = needed_map[pkgname] + for t in pkg.deps: + for d in pkg.deps[t]: + if split_package(d)[0] not in needed_map and not args.ignore_invalid: + msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper()) + if args.warn_invalid: + eprint("WARNING: %s" % msg) + else: + raise Exception(msg) + + node_map = {} + + # Convert all packages to target-specific nodes + for pkgname in needed_map: + pkg = needed_map[pkgname] + for target in pkg.deps: + if pkg.deps[target]: + node = Node(pkgname, target, pkg.section) + node_map[node.fqname] = node + + # Ensure all referenced dependencies exist as a basic node + for pkgname in needed_map: + pkg = needed_map[pkgname] + for target in pkg.deps: + for dep in pkg.deps[target]: + dfq = dep if dep.find(":") != -1 else "%s:target" % dep + if dfq not in node_map: + (dfq_p, dfq_t) = split_package(dfq) + if dfq_p in packages: + dpkg = packages[dfq_p] + node_map[dfq] = Node(dfq_p, dfq_t, dpkg.section) + elif not args.ignore_invalid: + raise Exception("Invalid package! Package %s cannot be found for this PROJECT/DEVICE/ARCH" % dfq_p) + + # To each target-specific node, add the correspnding + # target-specific dependency nodes ("edges") + for name in node_map: + node = node_map[name] + if node.name not in needed_map: + if args.warn_invalid: + continue + else: + raise Exception("Invalid package! Package %s cannot be found for this PROJECT/DEVICE/ARCH" % node.name) + for dep in needed_map[node.name].deps[node.target]: + dfq = dep if dep.find(":") != -1 else "%s:target" % dep + if dfq in node_map: + node.addEdge(node_map[dfq]) + + return node_map + +#--------------------------------------------- +parser = argparse.ArgumentParser(description="Generate package dependency list for the requested build/install packages. \ + Package data will be read from stdin in JSON format.", \ + formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=25,width=90)) + +parser.add_argument("-b", "--build", nargs="+", metavar="PACKAGE", required=True, \ + help="Space-separated list of build trigger packages, either for host or target. Required property - specify at least one package.") + +parser.add_argument("--warn-invalid", action="store_true", \ + help="Warn about invalid/missing dependency packages, perhaps excluded by a PKG_ARCH incompatability. Default is to abort.") + +parser.add_argument("--no-reorder", action="store_true", default="True", \ + help="Do not resequence steps based on dependencies. This is the default.") + +parser.add_argument("--reorder", action="store_false", dest="no_reorder", \ + help="Disable --no-reorder and resequence packages to try and reduce stalls etc.") + +parser.add_argument("--show-wants", action="store_true", \ + help="Output \"wants\" dependencies for each step.") + +parser.add_argument("--hide-wants", action="store_false", dest="show_wants", default="True", \ + help="Disable --show-wants.") + +parser.add_argument("--ignore-invalid", action="store_true", \ + help="Ignore invalid packages.") + +args = parser.parse_args() + +ALL_PACKAGES = loadPackages() + +loaded = len(ALL_PACKAGES) + +REQUIRED_PKGS = processPackages(args, ALL_PACKAGES, args.build) + +# Output list of packages to build/install +built_pkgs = [] +steps = [] + +for step in get_build_steps(args, REQUIRED_PKGS, args.build, built_pkgs): + steps.append(step) + +eprint("Packages loaded : %d" % loaded) +eprint("Build trigger(s): %d [%s]" % (len(args.build), " ".join(args.build))) +eprint("Package steps : %d" % len(steps)) +eprint("") + +# Output build/install steps +if args.show_wants: + for step in steps: + wants = [] + node = (REQUIRED_PKGS[step[1]]) + for e in node.edges: + wants.append(e.fqname) + print("%-7s %-25s (wants: %s)" % (step[0], step[1].replace(":target",""), ", ".join(wants).replace(":target",""))) +else: + for step in steps: + print("%-7s %s" % (step[0], step[1].replace(":target",""))) diff --git a/scripts/image b/scripts/image index 57febd43b6..de595fe5a4 100755 --- a/scripts/image +++ b/scripts/image @@ -1,28 +1,22 @@ #!/bin/bash -# SPDX-License-Identifier: GPL-2.0-or-later -# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) -# Copyright (C) 2016-present Team LibreELEC (https://libreelec.tv) +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST . config/options "" +. config/multithread . config/show_config show_config save_build_config +# Setup both toolchain cmake configs to avoid potentially racy behaviour later. +# Use a fork for host to isolate any variable modifications. +( setup_toolchain host ) setup_toolchain target -$SCRIPTS/checkdeps -$SCRIPTS/build toolchain -$SCRIPTS/build squashfs:host -$SCRIPTS/build dosfstools:host -$SCRIPTS/build fakeroot:host -$SCRIPTS/build kmod:host -$SCRIPTS/build mtools:host -$SCRIPTS/build populatefs:host - if [ -n "$CUSTOM_GIT_HASH" ]; then GIT_HASH="$CUSTOM_GIT_HASH" else @@ -82,34 +76,41 @@ fi echo "$IMAGE_NAME" > $BUILD/BUILD_FILENAME -# setup fakeroot +# Setup fakeroot rm -rf $FAKEROOT_SCRIPT # remove $FAKEROOT_SCRIPT if it exist touch $FAKEROOT_SCRIPT # create an empty $FAKEROOT_SCRIPT chmod +x $FAKEROOT_SCRIPT # make $FAKEROOT_SCRIPT executable echo "chown -R 0:0 $INSTALL" >> $FAKEROOT_SCRIPT -# clean old install dirs +# Clean old install dirs rm -rf $INSTALL rm -rf $STAMPS_INSTALL mkdir -p $INSTALL -# create baselayout +# Create base layout of LibreELEC read-only file system for directory in etc dev proc run sys tmp usr var flash storage; do mkdir -p $INSTALL/$directory done -ln -sf /var/media $INSTALL/media -ln -sf /usr/lib $INSTALL/lib -ln -sf /usr/bin $INSTALL/bin -ln -sf /usr/sbin $INSTALL/sbin + +# Build image contents +MTADDONBUILD=no start_multithread_build image || die "Parallel build failure - see log for details. Time of failure: $(date)" + +echo "Successful build, creating image..." >&2 + +# Create legacy sym links +ln -sfn /var/media $INSTALL/media +ln -sfn /usr/lib $INSTALL/lib +ln -sfn /usr/bin $INSTALL/bin +ln -sfn /usr/sbin $INSTALL/sbin if [ "$TARGET_ARCH" = "x86_64" ]; then - ln -s /usr/lib $INSTALL/lib64 - ln -s /usr/lib $INSTALL/usr/lib64 + ln -sfn /usr/lib $INSTALL/lib64 + ln -sfn /usr/lib $INSTALL/usr/lib64 fi echo "$TARGET_VERSION" > $INSTALL/etc/release -# create /etc/os-release +# Create /etc/os-release echo -e "NAME=\"$DISTRONAME\"" > $INSTALL/etc/os-release echo -e "VERSION=\"$LIBREELEC_VERSION\"" >> $INSTALL/etc/os-release echo -e "ID=\"libreelec\"" >> $INSTALL/etc/os-release @@ -126,7 +127,7 @@ echo -e "LIBREELEC_PROJECT=\"$PROJECT\"" >> $INSTALL/etc/os-release [ -n "$BUILDER_NAME" ] && echo -e "BUILDER_NAME=\"$BUILDER_NAME\"" >> $INSTALL/etc/os-release [ -n "$BUILDER_VERSION" ] && echo -e "BUILDER_VERSION=\"$BUILDER_VERSION\"" >> $INSTALL/etc/os-release -# create /etc/issue +# Create /etc/issue echo "$GREETING0" > $INSTALL/etc/issue echo "$GREETING1" >> $INSTALL/etc/issue echo "$GREETING2" >> $INSTALL/etc/issue @@ -136,64 +137,10 @@ echo "$DISTRONAME ($LIBREELEC_BUILD): $LIBREELEC_VERSION ($LIBREELEC_ARCH)" >> $ ln -sf /etc/issue $INSTALL/etc/motd -# populate base system... -$SCRIPTS/install libc -$SCRIPTS/install gcc - -$SCRIPTS/install linux -export _CACHED_KERNEL_MODULE_DIR="$(get_module_dir)" - -$SCRIPTS/install linux-drivers -$SCRIPTS/install linux-firmware -$SCRIPTS/install $BOOTLOADER -$SCRIPTS/install busybox -$SCRIPTS/install util-linux -$SCRIPTS/install corefonts -$SCRIPTS/install network - -# Graphic support -[ ! "$DISPLAYSERVER" = "no" ] && $SCRIPTS/install $DISPLAYSERVER - -# Multimedia support -[ ! "$MEDIACENTER" = "no" ] && $SCRIPTS/install mediacenter - -# Sound support -[ "$ALSA_SUPPORT" = "yes" ] && $SCRIPTS/install alsa - -# Automounter support -[ "$UDEVIL" = "yes" ] && $SCRIPTS/install udevil - -# EXFAT support -[ "$EXFAT" = "yes" ] && $SCRIPTS/install fuse-exfat - -# NTFS 3G support -[ "$NTFS3G" = "yes" ] && $SCRIPTS/install ntfs-3g_ntfsprogs - -# Remote support -[ "$REMOTE_SUPPORT" = "yes" ] && $SCRIPTS/install remote - -# Install miscellaneous packages -$SCRIPTS/install misc-packages - -# Virtual image creation support -[ "$PROJECT" = "Generic" ] && $SCRIPTS/install virtual - -# Installer support -[ "$INSTALLER_SUPPORT" = "yes" ] && $SCRIPTS/install installer - -# Devtools... (not for Release) -[ "$TESTING" = "yes" ] && $SCRIPTS/install testing - -# Install gdb in all builds, including releases -$SCRIPTS/install debug - -# OEM packages -[ "$OEM_SUPPORT" = "yes" ] && $SCRIPTS/install oem - -# copy PROJECT related files to filesystem +# Copy PROJECT related files to filesystem if [ -d "$PROJECT_DIR/$PROJECT/filesystem" ]; then cp -PR $PROJECT_DIR/$PROJECT/filesystem/* $INSTALL - # install project specific systemd services + # Install project specific systemd services for service in $PROJECT_DIR/$PROJECT/filesystem/usr/lib/systemd/system/*.service; do if [ -f "$service" ]; then enable_service $(basename $service) @@ -201,10 +148,10 @@ if [ -d "$PROJECT_DIR/$PROJECT/filesystem" ]; then done fi -# copy DEVICE related files to filesystem +# Copy DEVICE related files to filesystem if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem" ]; then cp -PR $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/* $INSTALL - # install device specific systemd services + # Install device specific systemd services for service in $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/usr/lib/systemd/system/*.service; do if [ -f "$service" ]; then enable_service $(basename $service) @@ -212,29 +159,29 @@ if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem" ]; th done fi -# run depmod for base overlay modules +# Run depmod for base overlay modules MODVER=$(basename $(ls -d $INSTALL/usr/lib/kernel-overlays/base/lib/modules/*)) find $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/ -name *.ko | \ sed -e "s,$INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/,," \ > $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/modules.order -$TOOLCHAIN/bin/depmod -b $INSTALL/usr/lib/kernel-overlays/base -a -e -F "$BUILD/linux-$(kernel_version)/System.map" $MODVER +$TOOLCHAIN/bin/depmod -b $INSTALL/usr/lib/kernel-overlays/base -a -e -F "$BUILD/linux-$(kernel_version)/System.map" $MODVER 2>&1 -# strip kernel modules +# Strip kernel modules for MOD in $(find $INSTALL/usr/lib/kernel-overlays/ -type f -name *.ko); do ${TARGET_KERNEL_PREFIX}strip --strip-debug $MOD done -# symlink overlayed modules to /usr/lib/modules +# Symlink overlayed modules to /usr/lib/modules ln -sT /var/lib/modules $INSTALL/usr/lib/modules -# symlink overlayed firmware to /usr/lib/firmware +# Symlink overlayed firmware to /usr/lib/firmware ln -sT /var/lib/firmware $INSTALL/usr/lib/firmware -# make target dir +# Make target dir mkdir -p $TARGET_IMG rm -rf $TARGET_IMG/$IMAGE_NAME.kernel -# copy kernel to target dir +# Copy kernel to target dir cp -PR $BUILD/linux-$(kernel_version)/arch/$TARGET_KERNEL_ARCH/boot/$KERNEL_TARGET $TARGET_IMG/$IMAGE_NAME.kernel chmod 0644 $TARGET_IMG/$IMAGE_NAME.kernel @@ -249,28 +196,28 @@ if [ -z "$SQUASHFS_COMPRESSION_OPTION" ]; then fi fi -# create squashfs file, default to gzip if no compression configured +# Create squashfs file, default to gzip if no compression configured echo "rm -rf \"$TARGET_IMG/$IMAGE_NAME.system\"" >> $FAKEROOT_SCRIPT echo "$TOOLCHAIN/bin/mksquashfs \"$BUILD/image/system\" \"$TARGET_IMG/$IMAGE_NAME.system\" -noappend -comp ${SQUASHFS_COMPRESSION:-gzip} ${SQUASHFS_COMPRESSION_OPTION}" >> $FAKEROOT_SCRIPT -# run fakeroot +# Run fakeroot $TOOLCHAIN/bin/fakeroot -- $FAKEROOT_SCRIPT rm -rf $FAKEROOT_SCRIPT -# set permissions +# Set permissions chmod 0644 $TARGET_IMG/$IMAGE_NAME.system if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ]; then RELEASE_DIR="target/$IMAGE_NAME" - # cleanup + # Cleanup rm -rf $RELEASE_DIR - # create release dir + # Create release dir mkdir -p $RELEASE_DIR - # remove n previous created release image + # Remove any previously created release images rm -rf $TARGET_IMG/$IMAGE_NAME.img.gz if [ -n "$BOOTLOADER" ]; then @@ -302,27 +249,27 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/target/SYSTEM cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/target/KERNEL - # create md5sum's + # Create md5sum's ( cd $RELEASE_DIR; md5sum -t target/SYSTEM > target/SYSTEM.md5; md5sum -t target/KERNEL > target/KERNEL.md5; ) - # create target directory + # Create target directory mkdir -p $TARGET_IMG - # remove an previous created release tarball + # Remove any previously created release tarballs rm -rf $TARGET_IMG/$IMAGE_NAME.tar - # create release tarball + # Create release tarball tar cf $TARGET_IMG/$IMAGE_NAME.tar -C target $IMAGE_NAME - # create sha256 checksum of tarball + # Create sha256 checksum of tarball ( cd $TARGET_IMG sha256sum ${IMAGE_NAME}.tar > ${IMAGE_NAME}.tar.sha256 ) - # create image files if requested + # Create image files if requested if [[ ( "$1" = "amlpkg" || "$1" = "noobs" || "$1" = "mkimage" ) && -n "$BOOTLOADER" ]]; then # INSTALL_SRC_DIR can be board specific if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/install" ]; then @@ -331,7 +278,7 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] INSTALL_SRC_DIR="$PROJECT_DIR/$PROJECT/install" fi - # variables used in image script must be passed + # Variables used in image script must be passed env \ PATH="$PATH:/sbin" \ ROOT="$ROOT" \ @@ -360,50 +307,50 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] $SCRIPTS/mkimage fi - # cleanup release dir + # Cleanup release dir rm -rf $RELEASE_DIR - # create WeTek Play (Amlogic) ZIP update and auto-install packages if requested + # Create WeTek Play (Amlogic) ZIP update and auto-install packages if requested if [ "$1" = "amlpkg" ]; then echo "Creating Amlogic ZIP update package" AML_PKG_DIR="$RELEASE_DIR/ampl-pkg" - # create package directory + # Create package directory mkdir -p "$AML_PKG_DIR" - # copy system and kernel images + # Copy system and kernel images mkdir -p "$AML_PKG_DIR/system" cp $TARGET_IMG/$IMAGE_NAME.system $AML_PKG_DIR/system/SYSTEM cp $TARGET_IMG/$IMAGE_NAME.kernel $AML_PKG_DIR/KERNEL - # copy update-binary and updater-script + # Copy update-binary and updater-script META_INF_DIR="$AML_PKG_DIR/META-INF/com/google/android" mkdir -p "$META_INF_DIR" cp $INSTALL_SRC_DIR/update-binary $META_INF_DIR cp $INSTALL_SRC_DIR/updater-script $META_INF_DIR - # copy other files if any + # Copy other files if any if [ -d "$INSTALL_SRC_DIR/files" ]; then cp -PR $INSTALL_SRC_DIR/files/* $AML_PKG_DIR fi - # copy device tree image if any + # Copy device tree image if any if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then cp "$INSTALL/usr/share/bootloader/dtb.img" $AML_PKG_DIR/dtb.img fi - # create the update package + # Create the update package pushd "$AML_PKG_DIR" > /dev/null zip -rq update.zip * - # sign the update package + # Sign the update package echo "Signing the update package" mkdir -p sign SIGNAPK_DIR="$ROOT/tools/signapk" java -Xmx1024m -jar $SIGNAPK_DIR/signapk.jar -w $SIGNAPK_DIR/testkey.x509.pem $SIGNAPK_DIR/testkey.pk8 update.zip sign/$IMAGE_NAME-update.zip - # create the auto-install package + # Create the auto-install package echo "Creating Amlogic ZIP auto-install package" pushd sign > /dev/null echo --update_package=/sdcard/$IMAGE_NAME-update.zip > factory_update_param.aml @@ -417,14 +364,14 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] cp $INSTALL_SRC_DIR/files/aml_autoscript . fi - # copy device tree image if any + # Copy device tree image if any if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then cp "$INSTALL/usr/share/bootloader/dtb.img" . fi zip -q $TARGET_IMG/$IMAGE_NAME.zip * - # create sha256 checksum of zip + # Create sha256 checksum of zip ( cd $TARGET_IMG sha256sum ${IMAGE_NAME}.zip > ${IMAGE_NAME}.zip.sha256 ) @@ -440,7 +387,7 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] # eg. LibreELEC_RPi, LibreELEC_RPi2 etc. NOOBS_DISTRO="${DISTRONAME}_${DEVICE:-$PROJECT}" - # create release dir + # Create release dir mkdir -p $RELEASE_DIR/${NOOBS_DISTRO} if [ -f $DISTRO_DIR/$DISTRO/${DISTRONAME}_40x40.png ]; then @@ -475,7 +422,7 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] -e "s%@SYSTEM_SIZE@%$SYSTEM_SIZE%g" \ -i $RELEASE_DIR/${NOOBS_DISTRO}/partitions.json - # create System dir + # Create System dir mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System BOOTLOADER_DIR="$(get_pkg_directory "$BOOTLOADER")" @@ -483,14 +430,14 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] cp -PR $BOOTLOADER_DIR/files/3rdparty/bootloader/* $RELEASE_DIR/${NOOBS_DISTRO}/System fi - # copy Bootloader + # Copy Bootloader cp -PR $BUILD/bcm2835-bootloader-*/LICENCE* $RELEASE_DIR/${NOOBS_DISTRO}/System/ cp -PR $BUILD/bcm2835-bootloader-*/bootcode.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/ cp -PR $BUILD/bcm2835-bootloader-*/fixup_x.dat $RELEASE_DIR/${NOOBS_DISTRO}/System/fixup.dat cp -PR $BUILD/bcm2835-bootloader-*/start_x.elf $RELEASE_DIR/${NOOBS_DISTRO}/System/start.elf [ -f $BUILD/bcm2835-bootloader-*/dt-blob.bin ] && cp -PR $BUILD/bcm2835-bootloader-*/dt-blob.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/dt-blob.bin - # copy system files + # Copy system files cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/${NOOBS_DISTRO}/System/SYSTEM cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/${NOOBS_DISTRO}/System/kernel.img @@ -507,45 +454,45 @@ if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ] fi done - # create md5sum's + # Create md5sum's ( cd $RELEASE_DIR/${NOOBS_DISTRO}/System; md5sum -t SYSTEM > SYSTEM.md5; md5sum -t kernel.img > kernel.img.md5; ) - # copy additional files + # Copy additional files mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses cp $ROOT/licenses/* $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses - # create Storage dir + # Create Storage dir mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/Storage - # remove any previously created release tarball + # Remove any previously created release tarball rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz - # create filesystem tarballs + # Create filesystem tarballs tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/System/ . tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/Storage/ . - # remove filesystem dirs + # Remove filesystem dirs rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage - # remove any previously created release tarball + # Remove any previously created release tarball rm -rf $TARGET_IMG/${IMAGE_NAME}-$1.tar - # create release tarball + # Create release tarball tar cf $TARGET_IMG/${IMAGE_NAME}-$1.tar -C $TARGET_IMG ${IMAGE_NAME}-$1 - # create sha256 checksum of tarball + # Create sha256 checksum of tarball ( cd $TARGET_IMG sha256sum ${IMAGE_NAME}-$1.tar > ${IMAGE_NAME}-$1.tar.sha256 ) fi if [ -d $RELEASE_DIR ]; then - # cleanup release dir + # Cleanup release dir rm -rf $RELEASE_DIR fi fi diff --git a/scripts/image_st b/scripts/image_st new file mode 100755 index 0000000000..bd4decf390 --- /dev/null +++ b/scripts/image_st @@ -0,0 +1,551 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0-or-later +# Copyright (C) 2009-2016 Stephan Raue (stephan@openelec.tv) +# Copyright (C) 2016-present Team LibreELEC (https://libreelec.tv) + +unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST + +. config/options "" +. config/show_config + +show_config +save_build_config + +setup_toolchain target + +$SCRIPTS/checkdeps +$SCRIPTS/build toolchain +$SCRIPTS/build squashfs:host +$SCRIPTS/build dosfstools:host +$SCRIPTS/build fakeroot:host +$SCRIPTS/build kmod:host +$SCRIPTS/build mtools:host +$SCRIPTS/build populatefs:host + +if [ -n "$CUSTOM_GIT_HASH" ]; then + GIT_HASH="$CUSTOM_GIT_HASH" +else + GIT_HASH=$(git rev-parse HEAD) +fi + +if [ "$LIBREELEC_VERSION" = "devel" ]; then + GIT_ABBREV=${GIT_HASH:0:7} + DEVEL_VERSION=$LIBREELEC_VERSION + case "$BUILD_PERIODIC" in + nightly) LIBREELEC_VERSION=nightly-$(date +%Y%m%d)-$GIT_ABBREV;; + daily) LIBREELEC_VERSION=daily-$(date +%Y%j)-$GIT_ABBREV;; + weekly) LIBREELEC_VERSION=weekly-$(date +%G%V)-$GIT_ABBREV;; + monthly) LIBREELEC_VERSION=monthly-$(date +%Y%m)-$GIT_ABBREV;; + *) LIBREELEC_VERSION=devel-$(date +%Y%m%d%H%M%S)-$GIT_ABBREV;; + esac +fi + +# Get origin url, fix git:// and git@github.com: urls if necessary +ORIGIN_URL="$(git remote -v | awk '$1 == "origin" { print $2 }' | head -1 | sed 's#\.git$##;s#^git:#https:#;s#^git@github\.com:#https://github.com/#')" + +[ "${BUILDER_NAME,,}" = "official" ] && BUILDER_NAME= +if [ "$OFFICIAL" = "yes" ]; then + LIBREELEC_BUILD="official" +else + if [ -n "$BUILDER_NAME" ]; then + LIBREELEC_BUILD="$BUILDER_NAME" + else + LIBREELEC_BUILD="community" + fi +fi + +if [ -n "$CUSTOM_VERSION" ]; then + LIBREELEC_VERSION="$CUSTOM_VERSION" +fi + +LIBREELEC_ARCH="${DEVICE:-$PROJECT}.$TARGET_ARCH" +TARGET_VERSION="$LIBREELEC_ARCH-$LIBREELEC_VERSION" + +if [ -n "$CUSTOM_IMAGE_NAME" ]; then + IMAGE_NAME="$CUSTOM_IMAGE_NAME" +else + if [ "$DEVEL_VERSION" = "devel" ]; then + IMAGE_NAME="$DISTRONAME-$LIBREELEC_ARCH-$OS_VERSION-$LIBREELEC_VERSION" + else + IMAGE_NAME="$DISTRONAME-$TARGET_VERSION" + fi + + if [ -n "$UBOOT_SYSTEM" ] && [ "$UBOOT_SYSTEM" != "${DEVICE:-$PROJECT}" ]; then + IMAGE_NAME="$IMAGE_NAME-$UBOOT_SYSTEM" + fi +fi + +if [ -n "$IMAGE_SUFFIX" ]; then + IMAGE_NAME="$IMAGE_NAME-$IMAGE_SUFFIX" +fi + +echo "$IMAGE_NAME" > $BUILD/BUILD_FILENAME + +# setup fakeroot +rm -rf $FAKEROOT_SCRIPT # remove $FAKEROOT_SCRIPT if it exist +touch $FAKEROOT_SCRIPT # create an empty $FAKEROOT_SCRIPT +chmod +x $FAKEROOT_SCRIPT # make $FAKEROOT_SCRIPT executable +echo "chown -R 0:0 $INSTALL" >> $FAKEROOT_SCRIPT + +# clean old install dirs +rm -rf $INSTALL +rm -rf $STAMPS_INSTALL +mkdir -p $INSTALL + +# create baselayout +for directory in etc dev proc run sys tmp usr var flash storage; do + mkdir -p $INSTALL/$directory +done +ln -sfn /var/media $INSTALL/media +ln -sfn /usr/lib $INSTALL/lib +ln -sfn /usr/bin $INSTALL/bin +ln -sfn /usr/sbin $INSTALL/sbin + +if [ "$TARGET_ARCH" = "x86_64" ]; then + ln -sfn /usr/lib $INSTALL/lib64 + ln -sfn /usr/lib $INSTALL/usr/lib64 +fi + +echo "$TARGET_VERSION" > $INSTALL/etc/release + +# create /etc/os-release +echo -e "NAME=\"$DISTRONAME\"" > $INSTALL/etc/os-release +echo -e "VERSION=\"$LIBREELEC_VERSION\"" >> $INSTALL/etc/os-release +echo -e "ID=\"libreelec\"" >> $INSTALL/etc/os-release +echo -e "VERSION_ID=\"$OS_VERSION\"" >> $INSTALL/etc/os-release +echo -e "PRETTY_NAME=\"$DISTRONAME ($LIBREELEC_BUILD): $LIBREELEC_VERSION\"" >> $INSTALL/etc/os-release +echo -e "HOME_URL=\"https://libreelec.tv\"" >> $INSTALL/etc/os-release +echo -e "BUG_REPORT_URL=\"$ORIGIN_URL\"" >> $INSTALL/etc/os-release +echo -e "BUILD_ID=\"$GIT_HASH\"" >> $INSTALL/etc/os-release +echo -e "OPENELEC_ARCH=\"$LIBREELEC_ARCH\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_ARCH=\"$LIBREELEC_ARCH\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_BUILD=\"$LIBREELEC_BUILD\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_PROJECT=\"$PROJECT\"" >> $INSTALL/etc/os-release +[ -n "$DEVICE" ] && echo -e "LIBREELEC_DEVICE=\"$DEVICE\"" >> $INSTALL/etc/os-release +[ -n "$BUILDER_NAME" ] && echo -e "BUILDER_NAME=\"$BUILDER_NAME\"" >> $INSTALL/etc/os-release +[ -n "$BUILDER_VERSION" ] && echo -e "BUILDER_VERSION=\"$BUILDER_VERSION\"" >> $INSTALL/etc/os-release + +# create /etc/issue +echo "$GREETING0" > $INSTALL/etc/issue +echo "$GREETING1" >> $INSTALL/etc/issue +echo "$GREETING2" >> $INSTALL/etc/issue +echo "$GREETING3" >> $INSTALL/etc/issue +echo "$GREETING4" >> $INSTALL/etc/issue +echo "$DISTRONAME ($LIBREELEC_BUILD): $LIBREELEC_VERSION ($LIBREELEC_ARCH)" >> $INSTALL/etc/issue + +ln -sf /etc/issue $INSTALL/etc/motd + +# populate base system... +$SCRIPTS/install libc +$SCRIPTS/install gcc + +$SCRIPTS/install linux +export _CACHED_KERNEL_MODULE_DIR="$(get_module_dir)" + +$SCRIPTS/install linux-drivers +$SCRIPTS/install linux-firmware +$SCRIPTS/install $BOOTLOADER +$SCRIPTS/install busybox +$SCRIPTS/install util-linux +$SCRIPTS/install corefonts +$SCRIPTS/install network + +# Graphic support +[ ! "$DISPLAYSERVER" = "no" ] && $SCRIPTS/install $DISPLAYSERVER + +# Multimedia support +[ ! "$MEDIACENTER" = "no" ] && $SCRIPTS/install mediacenter + +# Sound support +[ "$ALSA_SUPPORT" = "yes" ] && $SCRIPTS/install alsa + +# Automounter support +[ "$UDEVIL" = "yes" ] && $SCRIPTS/install udevil + +# EXFAT support +[ "$EXFAT" = "yes" ] && $SCRIPTS/install fuse-exfat + +# NTFS 3G support +[ "$NTFS3G" = "yes" ] && $SCRIPTS/install ntfs-3g_ntfsprogs + +# Remote support +[ "$REMOTE_SUPPORT" = "yes" ] && $SCRIPTS/install remote + +# Install miscellaneous packages +$SCRIPTS/install misc-packages + +# Virtual image creation support +[ "$PROJECT" = "Generic" ] && $SCRIPTS/install virtual + +# Installer support +[ "$INSTALLER_SUPPORT" = "yes" ] && $SCRIPTS/install installer + +# Devtools... (not for Release) +[ "$TESTING" = "yes" ] && $SCRIPTS/install testing + +# Install gdb in all builds, including releases +$SCRIPTS/install debug + +# OEM packages +[ "$OEM_SUPPORT" = "yes" ] && $SCRIPTS/install oem + +# copy PROJECT related files to filesystem +if [ -d "$PROJECT_DIR/$PROJECT/filesystem" ]; then + cp -PR $PROJECT_DIR/$PROJECT/filesystem/* $INSTALL + # install project specific systemd services + for service in $PROJECT_DIR/$PROJECT/filesystem/usr/lib/systemd/system/*.service; do + if [ -f "$service" ]; then + enable_service $(basename $service) + fi + done +fi + +# copy DEVICE related files to filesystem +if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem" ]; then + cp -PR $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/* $INSTALL + # install device specific systemd services + for service in $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/usr/lib/systemd/system/*.service; do + if [ -f "$service" ]; then + enable_service $(basename $service) + fi + done +fi + +# run depmod for base overlay modules +MODVER=$(basename $(ls -d $INSTALL/usr/lib/kernel-overlays/base/lib/modules/*)) +find $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/ -name *.ko | \ + sed -e "s,$INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/,," \ + > $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/modules.order +$TOOLCHAIN/bin/depmod -b $INSTALL/usr/lib/kernel-overlays/base -a -e -F "$BUILD/linux-$(kernel_version)/System.map" $MODVER + +# strip kernel modules +for MOD in $(find $INSTALL/usr/lib/kernel-overlays/ -type f -name *.ko); do + ${TARGET_KERNEL_PREFIX}strip --strip-debug $MOD +done + +# symlink overlayed modules to /usr/lib/modules +ln -sT /var/lib/modules $INSTALL/usr/lib/modules + +# symlink overlayed firmware to /usr/lib/firmware +ln -sT /var/lib/firmware $INSTALL/usr/lib/firmware + +# make target dir +mkdir -p $TARGET_IMG +rm -rf $TARGET_IMG/$IMAGE_NAME.kernel + +# copy kernel to target dir +cp -PR $BUILD/linux-$(kernel_version)/arch/$TARGET_KERNEL_ARCH/boot/$KERNEL_TARGET $TARGET_IMG/$IMAGE_NAME.kernel +chmod 0644 $TARGET_IMG/$IMAGE_NAME.kernel + +# Set mksquashfs options for each compression method +if [ -z "$SQUASHFS_COMPRESSION_OPTION" ]; then + if [ "${SQUASHFS_COMPRESSION:-gzip}" = "gzip" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 9 -b 262144" + elif [ "$SQUASHFS_COMPRESSION" = "lzo" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 9 -b 524288" + elif [ "$SQUASHFS_COMPRESSION" = "zstd" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 22 -b 262144" + fi +fi + +# create squashfs file, default to gzip if no compression configured +echo "rm -rf \"$TARGET_IMG/$IMAGE_NAME.system\"" >> $FAKEROOT_SCRIPT +echo "$TOOLCHAIN/bin/mksquashfs \"$BUILD/image/system\" \"$TARGET_IMG/$IMAGE_NAME.system\" -noappend -comp ${SQUASHFS_COMPRESSION:-gzip} ${SQUASHFS_COMPRESSION_OPTION}" >> $FAKEROOT_SCRIPT + +# run fakeroot +$TOOLCHAIN/bin/fakeroot -- $FAKEROOT_SCRIPT +rm -rf $FAKEROOT_SCRIPT + +# set permissions +chmod 0644 $TARGET_IMG/$IMAGE_NAME.system + +if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ]; then + + RELEASE_DIR="target/$IMAGE_NAME" + + # cleanup + rm -rf $RELEASE_DIR + + # create release dir + mkdir -p $RELEASE_DIR + + # remove n previous created release image + rm -rf $TARGET_IMG/$IMAGE_NAME.img.gz + + if [ -n "$BOOTLOADER" ]; then + + BOOTLOADER_DIR="$(get_pkg_directory "$BOOTLOADER")" + + if [ -d $BOOTLOADER_DIR/files ]; then + cp -R $BOOTLOADER_DIR/files/* $RELEASE_DIR + fi + + if find_file_path bootloader/release $BOOTLOADER_DIR/release; then + echo "Running $FOUND_PATH" + . $FOUND_PATH + fi + fi + + cp $ROOT/README* $RELEASE_DIR + cp $ROOT/CHANGELOG* $RELEASE_DIR + echo "$TARGET_VERSION" > $RELEASE_DIR/RELEASE + + if [ ! "$MEDIACENTER" = "no" ]; then + echo "Kodi commit: $(get_pkg_version $MEDIACENTER)" >> $RELEASE_DIR/RELEASE + fi + + mkdir -p $RELEASE_DIR/licenses + cp $ROOT/licenses/* $RELEASE_DIR/licenses + + mkdir -p $RELEASE_DIR/target + cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/target/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/target/KERNEL + + # create md5sum's + ( cd $RELEASE_DIR; + md5sum -t target/SYSTEM > target/SYSTEM.md5; + md5sum -t target/KERNEL > target/KERNEL.md5; + ) + + # create target directory + mkdir -p $TARGET_IMG + + # remove an previous created release tarball + rm -rf $TARGET_IMG/$IMAGE_NAME.tar + + # create release tarball + tar cf $TARGET_IMG/$IMAGE_NAME.tar -C target $IMAGE_NAME + + # create sha256 checksum of tarball + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}.tar > ${IMAGE_NAME}.tar.sha256 + ) + + # create image files if requested + if [[ ( "$1" = "amlpkg" || "$1" = "noobs" || "$1" = "mkimage" ) && -n "$BOOTLOADER" ]]; then + # INSTALL_SRC_DIR can be board specific + if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/install" ]; then + INSTALL_SRC_DIR="$PROJECT_DIR/$PROJECT/devices/$DEVICE/install" + else + INSTALL_SRC_DIR="$PROJECT_DIR/$PROJECT/install" + fi + + # variables used in image script must be passed + env \ + PATH="$PATH:/sbin" \ + ROOT="$ROOT" \ + SCRIPTS="$SCRIPTS" \ + TOOLCHAIN="$TOOLCHAIN" \ + PROJECT_DIR="$PROJECT_DIR" \ + PROJECT="$PROJECT" \ + DEVICE="$DEVICE" \ + DISTRO="$DISTRO" \ + TARGET_IMG="$TARGET_IMG" \ + IMAGE_NAME="$IMAGE_NAME" \ + INSTALL_SRC_DIR="$INSTALL_SRC_DIR" \ + BOOTLOADER="$BOOTLOADER" \ + KERNEL_NAME="$KERNEL_NAME" \ + TARGET_KERNEL_ARCH="$TARGET_KERNEL_ARCH" \ + RELEASE_DIR=$RELEASE_DIR \ + UUID_STORAGE="$(uuidgen)" \ + DISTRO_BOOTLABEL="$DISTRO_BOOTLABEL" \ + DISTRO_DISKLABEL="$DISTRO_DISKLABEL" \ + UBOOT_SYSTEM="$UBOOT_SYSTEM" \ + UBOOT_VERSION="$UBOOT_VERSION" \ + EXTRA_CMDLINE="$EXTRA_CMDLINE" \ + SYSTEM_SIZE="$SYSTEM_SIZE" \ + SYSTEM_PART_START="$SYSTEM_PART_START" \ + OVA_SIZE="$OVA_SIZE" \ + $SCRIPTS/mkimage + fi + + # cleanup release dir + rm -rf $RELEASE_DIR + + # create WeTek Play (Amlogic) ZIP update and auto-install packages if requested + if [ "$1" = "amlpkg" ]; then + echo "Creating Amlogic ZIP update package" + + AML_PKG_DIR="$RELEASE_DIR/ampl-pkg" + + # create package directory + mkdir -p "$AML_PKG_DIR" + + # copy system and kernel images + mkdir -p "$AML_PKG_DIR/system" + cp $TARGET_IMG/$IMAGE_NAME.system $AML_PKG_DIR/system/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $AML_PKG_DIR/KERNEL + + # copy update-binary and updater-script + META_INF_DIR="$AML_PKG_DIR/META-INF/com/google/android" + mkdir -p "$META_INF_DIR" + cp $INSTALL_SRC_DIR/update-binary $META_INF_DIR + cp $INSTALL_SRC_DIR/updater-script $META_INF_DIR + + # copy other files if any + if [ -d "$INSTALL_SRC_DIR/files" ]; then + cp -PR $INSTALL_SRC_DIR/files/* $AML_PKG_DIR + fi + + # copy device tree image if any + if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then + cp "$INSTALL/usr/share/bootloader/dtb.img" $AML_PKG_DIR/dtb.img + fi + + # create the update package + pushd "$AML_PKG_DIR" > /dev/null + zip -rq update.zip * + + # sign the update package + echo "Signing the update package" + mkdir -p sign + SIGNAPK_DIR="$ROOT/tools/signapk" + java -Xmx1024m -jar $SIGNAPK_DIR/signapk.jar -w $SIGNAPK_DIR/testkey.x509.pem $SIGNAPK_DIR/testkey.pk8 update.zip sign/$IMAGE_NAME-update.zip + + # create the auto-install package + echo "Creating Amlogic ZIP auto-install package" + pushd sign > /dev/null + echo --update_package=/sdcard/$IMAGE_NAME-update.zip > factory_update_param.aml + echo --wipe_data >> factory_update_param.aml + echo --wipe_cache >> factory_update_param.aml + if [ -f "$INSTALL_SRC_DIR/files/recovery.img" ]; then + cp $INSTALL_SRC_DIR/files/recovery.img . + fi + + if [ -f $INSTALL_SRC_DIR/files/aml_autoscript ]; then + cp $INSTALL_SRC_DIR/files/aml_autoscript . + fi + + # copy device tree image if any + if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then + cp "$INSTALL/usr/share/bootloader/dtb.img" . + fi + + zip -q $TARGET_IMG/$IMAGE_NAME.zip * + + # create sha256 checksum of zip + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}.zip > ${IMAGE_NAME}.zip.sha256 + ) + + popd > /dev/null + popd > /dev/null + + elif [ "$1" = "noobs" ]; then + echo "Creating \"$1\" release tarball..." + + RELEASE_DIR="$TARGET_IMG/${IMAGE_NAME}-$1" + + # eg. LibreELEC_RPi, LibreELEC_RPi2 etc. + NOOBS_DISTRO="${DISTRONAME}_${DEVICE:-$PROJECT}" + + # create release dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO} + + if [ -f $DISTRO_DIR/$DISTRO/${DISTRONAME}_40x40.png ]; then + cp -PR $DISTRO_DIR/$DISTRO/${DISTRONAME}_40x40.png $RELEASE_DIR/${NOOBS_DISTRO}/${NOOBS_DISTRO}.png + else + cp -PR $DISTRO_DIR/$DISTRO/${DISTRONAME}.png $RELEASE_DIR/${NOOBS_DISTRO}/${NOOBS_DISTRO}.png + fi + cp -PR $ROOT/config/noobs/os.json $RELEASE_DIR/${NOOBS_DISTRO} + cp -PR $ROOT/config/noobs/partition_setup.sh $RELEASE_DIR/${NOOBS_DISTRO} + cp -PR $ROOT/config/noobs/partitions.json $RELEASE_DIR/${NOOBS_DISTRO} + if [ -d $DISTRO_DIR/$DISTRO/noobs/marketing ]; then + tar cf $RELEASE_DIR/${NOOBS_DISTRO}/marketing.tar -C $DISTRO_DIR/$DISTRO/noobs/marketing . + else + tar cf $RELEASE_DIR/${NOOBS_DISTRO}/marketing.tar -C $ROOT/config/noobs/marketing . + fi + cp $ROOT/README* $RELEASE_DIR/${NOOBS_DISTRO} + cp $ROOT/CHANGELOG $RELEASE_DIR/${NOOBS_DISTRO}/release_notes.txt + + sed -e "s%@DISTRONAME@%$DISTRONAME%g" \ + -e "s%@PROJECT@%${DEVICE:-$PROJECT}%g" \ + -e "s%@LIBREELEC_VERSION@%$LIBREELEC_VERSION%g" \ + -e "s%@RELEASE_DATE@%$(date +%F)%g" \ + -e "s%@KERNEL_VERSION@%$(kernel_version)%g" \ + -e "s%@DESCRIPTION@%$DESCRIPTION%g" \ + -e "s%@ROOT_PASSWORD@%$ROOT_PASSWORD%g" \ + -e "s%@NOOBS_SUPPORTED_MODELS@%$NOOBS_SUPPORTED_MODELS%g" \ + -e "s%@NOOBS_HEX@%$NOOBS_HEX%g" \ + -i $RELEASE_DIR/${NOOBS_DISTRO}/os.json + + sed -e "s%@DISTRONAME@%$DISTRONAME%g" \ + -e "s%@PROJECT@%${DEVICE:-$PROJECT}%g" \ + -e "s%@SYSTEM_SIZE@%$SYSTEM_SIZE%g" \ + -i $RELEASE_DIR/${NOOBS_DISTRO}/partitions.json + + # create System dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System + + BOOTLOADER_DIR="$(get_pkg_directory "$BOOTLOADER")" + if [ -d $BOOTLOADER_DIR/files/3rdparty/bootloader/ ]; then + cp -PR $BOOTLOADER_DIR/files/3rdparty/bootloader/* $RELEASE_DIR/${NOOBS_DISTRO}/System + fi + + # copy Bootloader + cp -PR $BUILD/bcm2835-bootloader-*/LICENCE* $RELEASE_DIR/${NOOBS_DISTRO}/System/ + cp -PR $BUILD/bcm2835-bootloader-*/bootcode.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/ + cp -PR $BUILD/bcm2835-bootloader-*/fixup_x.dat $RELEASE_DIR/${NOOBS_DISTRO}/System/fixup.dat + cp -PR $BUILD/bcm2835-bootloader-*/start_x.elf $RELEASE_DIR/${NOOBS_DISTRO}/System/start.elf + [ -f $BUILD/bcm2835-bootloader-*/dt-blob.bin ] && cp -PR $BUILD/bcm2835-bootloader-*/dt-blob.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/dt-blob.bin + + # copy system files + cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/${NOOBS_DISTRO}/System/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/${NOOBS_DISTRO}/System/kernel.img + + for dtb in $INSTALL/usr/share/bootloader/*.dtb; do + if [ -f $dtb ]; then + cp -PR $dtb $RELEASE_DIR/${NOOBS_DISTRO}/System + fi + done + + for overlay in $INSTALL/usr/share/bootloader/overlays/*; do + if [ -f $overlay ]; then + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System/overlays + cp -PR $overlay $RELEASE_DIR/${NOOBS_DISTRO}/System/overlays + fi + done + + # create md5sum's + ( cd $RELEASE_DIR/${NOOBS_DISTRO}/System; + md5sum -t SYSTEM > SYSTEM.md5; + md5sum -t kernel.img > kernel.img.md5; + ) + + # copy additional files + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses + cp $ROOT/licenses/* $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses + + # create Storage dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/Storage + + # remove any previously created release tarball + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz + + # create filesystem tarballs + tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/System/ . + tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/Storage/ . + + # remove filesystem dirs + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage + + # remove any previously created release tarball + rm -rf $TARGET_IMG/${IMAGE_NAME}-$1.tar + + # create release tarball + tar cf $TARGET_IMG/${IMAGE_NAME}-$1.tar -C $TARGET_IMG ${IMAGE_NAME}-$1 + + # create sha256 checksum of tarball + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}-$1.tar > ${IMAGE_NAME}-$1.tar.sha256 + ) + fi + + if [ -d $RELEASE_DIR ]; then + # cleanup release dir + rm -rf $RELEASE_DIR + fi +fi diff --git a/scripts/install b/scripts/install index cb8521ce49..375bb4c468 100755 --- a/scripts/install +++ b/scripts/install @@ -8,7 +8,7 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name" + die "usage: $0 package_name [parent_pkg]" fi if [ -z "${PKG_NAME}" ]; then @@ -29,29 +29,35 @@ if [ "${1//:/}" != "${1}" ]; then PACKAGE_NAME="${1%:*}" TARGET="${1#*:}" else - PACKAGE_NAME=$1 + PACKAGE_NAME="${1}" TARGET= fi [ -z "$TARGET" ] && TARGET="target" +PARENT_PKG="${2:-${PKG_NAME}:${TARGET}}" + +pkg_lock "${PACKAGE_NAME}:${TARGET}" "install" "${PARENT_PKG}" STAMP=$STAMPS_INSTALL/$PACKAGE_NAME/install_$TARGET +[ -f $STAMP ] && pkg_lock_status "UNLOCK" "${PACKAGE_NAME}:${TARGET}" "install" "already installed" [ -f $STAMP ] && exit 0 mkdir -p $STAMPS_INSTALL/$PACKAGE_NAME -$SCRIPTS/build $@ +$SCRIPTS/build "${1}" "${PARENT_PKG}" if [ "$TARGET" = target ] ; then for p in $PKG_DEPENDS_TARGET; do - $SCRIPTS/install $p + $SCRIPTS/install "${p}" "${PARENT_PKG}" done elif [ "$TARGET" = init ] ; then for p in $PKG_DEPENDS_INIT; do - $SCRIPTS/install $p + $SCRIPTS/install "${p}" "${PARENT_PKG}" done INSTALL=$BUILD/initramfs fi +pkg_lock_status "ACTIVE" "${PACKAGE_NAME}:${TARGET}" "install" + build_msg "CLR_INSTALL" "INSTALL" "${PACKAGE_NAME} $(print_color CLR_TARGET "(${TARGET})")" "indent" mkdir -p $INSTALL @@ -137,3 +143,5 @@ if [ "$TARGET" = target ] ; then fi touch $STAMP + +pkg_lock_status "UNLOCK" "${PACKAGE_NAME}:${TARGET}" "install" "installed" diff --git a/scripts/install_addon b/scripts/install_addon index 0564c472f3..e74d0bb85e 100755 --- a/scripts/install_addon +++ b/scripts/install_addon @@ -5,6 +5,9 @@ . config/options $1 +# Make sure we have a value for STRIP +setup_toolchain target + [ -n "$2" ] && PKG_ADDON_ID=$2 ADDON_DIRECTORY="$ADDON_BUILD/$PKG_ADDON_ID" @@ -14,3 +17,62 @@ mkdir -p "$ADDON_DIRECTORY" install_addon_files "$ADDON_DIRECTORY" debug_strip "$ADDON_DIRECTORY" + +# pack_addon() +ADDON_INSTALL_DIR="$TARGET_IMG/$ADDONS/$ADDON_VERSION/${DEVICE:-$PROJECT}/$TARGET_ARCH/$PKG_ADDON_ID" +ADDONVER="$(xmlstarlet sel -t -v "/addon/@version" $ADDON_BUILD/$PKG_ADDON_ID/addon.xml)" + +if [ -f $ADDON_INSTALL_DIR/$PKG_ADDON_ID-$ADDONVER.zip ]; then + if [ "$ADDON_OVERWRITE" = "yes" ]; then + rm $ADDON_INSTALL_DIR/$PKG_ADDON_ID-$ADDONVER.zip + else + build_msg "CLR_WARNING" "*** WARNING: ${PKG_ADDON_ID}-${ADDONVER}.zip already exists. Not overwrittng it. ***" + exit 0 + fi +fi + +cd $ADDON_BUILD +build_msg "CLR_INFO" "*** compressing addon ${PKG_ADDON_ID} ... ***" +$TOOLCHAIN/bin/7za a -l -mx9 -bsp0 -bso0 -tzip $PKG_ADDON_ID-$ADDONVER.zip $PKG_ADDON_ID +cd - &>/dev/null + +mkdir -p $ADDON_INSTALL_DIR +cp $ADDON_BUILD/$PKG_ADDON_ID-$ADDONVER.zip $ADDON_INSTALL_DIR +if [ -f $ADDON_BUILD/$PKG_ADDON_ID/changelog.txt ]; then + cp $ADDON_BUILD/$PKG_ADDON_ID/changelog.txt $ADDON_INSTALL_DIR/changelog-$ADDONVER.txt +fi +if [ -f $ADDON_BUILD/$PKG_ADDON_ID/resources/icon.png ]; then + mkdir -p $ADDON_INSTALL_DIR/resources + cp $ADDON_BUILD/$PKG_ADDON_ID/resources/icon.png $ADDON_INSTALL_DIR/resources/icon.png +fi + +# workaround for kodi pvr addons +if [ -f $ADDON_BUILD/$PKG_ADDON_ID/icon.png ]; then + cp $ADDON_BUILD/$PKG_ADDON_ID/icon.png $ADDON_INSTALL_DIR/icon.png +fi + +if [ -f $ADDON_BUILD/$PKG_ADDON_ID/resources/fanart.png ]; then + mkdir -p $ADDON_INSTALL_DIR/resources + cp $ADDON_BUILD/$PKG_ADDON_ID/resources/fanart.png $ADDON_INSTALL_DIR/resources/fanart.png +fi +for f in $ADDON_BUILD/$PKG_ADDON_ID/resources/screenshot-*.{jpg,png}; do + if [ -f "$f" ]; then + mkdir -p $ADDON_INSTALL_DIR/resources + cp $f $ADDON_INSTALL_DIR/resources + fi +done + +# Jenkins add-on build +if [ "$ADDON_JENKINS" = "yes" ]; then + ADDON_JENKINS_DIR="$TARGET_IMG/jenkins" + ADDON_JENKINS_ADDON_NAME="$ADDON_VERSION-${DEVICE:-$PROJECT}-$TARGET_ARCH-$PKG_ADDON_ID-$ADDONVER" + mkdir -p "$ADDON_JENKINS_DIR" + cd $ADDON_INSTALL_DIR + $TOOLCHAIN/bin/7za a -l -mx0 -bsp0 -bso0 -tzip $ADDON_JENKINS_DIR/$ADDON_JENKINS_ADDON_NAME.zip $PKG_ADDON_ID-$ADDONVER.zip resources/ + ( cd $ADDON_JENKINS_DIR + sha256sum $ADDON_JENKINS_ADDON_NAME.zip > $ADDON_JENKINS_ADDON_NAME.zip.sha256 + ) + build_msg "CLR_INFO" "*** creating ${ADDON_JENKINS_ADDON_NAME}.zip for Jenkins complete ***" +else + build_msg "CLR_INFO" "*** creating ${PKG_ADDON_ID} complete ***" +fi diff --git a/scripts/unpack b/scripts/unpack index 5da8a4ff77..fd7bbf1da0 100755 --- a/scripts/unpack +++ b/scripts/unpack @@ -7,14 +7,19 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name" + die "usage: $0 package_name [parent_pkg]" fi if [ -z "${PKG_NAME}" ]; then die "$(print_color CLR_ERROR "${1}: no package.mk file found")" fi +PARENT_PKG="${2:-${PKG_NAME}}" -$SCRIPTS/get $1 +pkg_lock "${PKG_NAME}" "unpack" "${PARENT_PKG}" + +pkg_lock_status "ACTIVE" "${PKG_NAME}" "unpack" + +$SCRIPTS/get "${PKG_NAME}" STAMP="$PKG_BUILD/.libreelec-unpack" @@ -22,13 +27,13 @@ mkdir -p $BUILD # Perform a wildcard match on the package to ensure old versions are cleaned too PKG_DEEPHASH= -for i in $BUILD/$1-*; do +for i in $BUILD/${PKG_NAME}-*; do if [ -d $i -a -f "$i/.libreelec-unpack" ] ; then . "$i/.libreelec-unpack" - if [ "$STAMP_PKG_NAME" = "$1" ]; then + if [ "$STAMP_PKG_NAME" = "${PKG_NAME}" ]; then [ -z "${PKG_DEEPHASH}" ] && PKG_DEEPHASH=$(calculate_stamp) if [ ! "$PKG_DEEPHASH" = "$STAMP_PKG_DEEPHASH" ] ; then - $SCRIPTS/clean $1 + $SCRIPTS/clean "${PKG_NAME}" fi fi fi @@ -36,13 +41,21 @@ done if [ -d "$PKG_BUILD" -a ! -f "$STAMP" ]; then # stale pkg build dir - $SCRIPTS/clean $1 + $SCRIPTS/clean "${PKG_NAME}" fi -[ -f "$STAMP" ] && exit 0 +if [ -f "$STAMP" ]; then + pkg_lock_status "UNLOCK" "${PKG_NAME}" "unpack" "already unpacked" + exit 0 +fi -if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then - build_msg "CLR_UNPACK" "UNPACK" "${1}" "indent" +if [ -d "$SOURCES/${PKG_NAME}" -o -d "$PKG_DIR/sources" ]; then + build_msg "CLR_UNPACK" "UNPACK" "${PKG_NAME}" "indent" + + # unpack into a unique location as unpacking into a single ${BUILD} directory is not thread-safe + PKG_UNPACK_DIR="${BUILD}/.unpack/${PKG_NAME}" + rm -fr "${PKG_UNPACK_DIR}" + mkdir -p "${PKG_UNPACK_DIR}" pkg_call_exists pre_unpack && pkg_call pre_unpack @@ -50,18 +63,29 @@ if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then pkg_call unpack else if [ -n "$PKG_URL" ]; then - $SCRIPTS/extract $1 $BUILD + ${SCRIPTS}/extract "${PKG_NAME}" "${PKG_UNPACK_DIR}" fi fi - if [ ! -d "$PKG_BUILD" ]; then - if [ -n "$PKG_SOURCE_DIR" ]; then - mv $BUILD/$PKG_SOURCE_DIR "$PKG_BUILD" - elif [ -d "$PKG_BUILD"* ]; then - mv "$PKG_BUILD"* "$PKG_BUILD" + _pkg_build="$(basename "${PKG_BUILD}")" + [ -d "${PKG_UNPACK_DIR}/${_pkg_build}"* ] && mv "${PKG_UNPACK_DIR}/${_pkg_build}"* "${PKG_BUILD}" + + if [ ! -d "${PKG_BUILD}" ]; then + if [ -n "${PKG_SOURCE_DIR}" ]; then + if [ -d "${PKG_UNPACK_DIR}"/${PKG_SOURCE_DIR} ]; then + mv "${PKG_UNPACK_DIR}"/${PKG_SOURCE_DIR} "${PKG_BUILD}" + else + # fallback + mv "${BUILD}"/${PKG_SOURCE_DIR} "${PKG_BUILD}" + fi + elif [ -d "${PKG_BUILD}"* ]; then + mv "${PKG_BUILD}"* "${PKG_BUILD}" fi fi + # cleanup + rm -fr "${PKG_UNPACK_DIR}" + if [ -d "$PKG_DIR/sources" ]; then [ ! -d "$PKG_BUILD" ] && mkdir -p "$PKG_BUILD" cp -PRf "$PKG_DIR/sources/"* "$PKG_BUILD" @@ -161,10 +185,12 @@ fi if [ "$PKG_SECTION" != "virtual" ]; then mkdir -p "$PKG_BUILD" - rm -f $STAMPS/$1/build_* + rm -f $STAMPS/${PKG_NAME}/build_* PKG_DEEPHASH=$(calculate_stamp) for i in PKG_NAME PKG_DEEPHASH; do echo "STAMP_$i=\"${!i}\"" >> $STAMP done fi + +pkg_lock_status "UNLOCK" "${PKG_NAME}" "unpack" "unpacked" diff --git a/tools/dashboard b/tools/dashboard new file mode 100755 index 0000000000..3e78450e97 --- /dev/null +++ b/tools/dashboard @@ -0,0 +1,19 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +. config/options "" + +while [ : ]; do + echo "Waiting for build to start..." + while [ : ]; do + pid="$(cat "${THREAD_CONTROL}/pid" 2>/dev/null || true)" + [ -n "${pid}" ] && ps -p ${pid} &>/dev/null && break + sleep 1.0 + done + + tail -Fn+0 --pid=${pid} ${THREAD_CONTROL}/status 2>/dev/null + + echo +done diff --git a/tools/mtstats.py b/tools/mtstats.py new file mode 100755 index 0000000000..5c22ea394d --- /dev/null +++ b/tools/mtstats.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +import sys, os, codecs, datetime, time + +EPOCH = datetime.datetime(1970, 1, 1) + +class HistoryEvent: + def __init__(self, event): + # squash spaces, then split by space + items = ' '.join(event.replace("\n", "").split()).split(" ") + + self.datetime = "%s %s" % (items[0], items[1][:-4]) + (self.slot, self.seq) = items[3][1:-1].split("/") + self.status = items[4] + self.task = items[5] if len(items) > 5 else "" + self.secs = None + + def __repr__(self): + return "%s; %s; %s; %s; %s" % (self.datetime, self.slot, self.seq, self.status, self.task) + + def get_time_secs(self): + if self.secs == None: + self.secs = (datetime.datetime.strptime(self.datetime, "%Y-%m-%d %H:%M:%S.%f") - EPOCH).total_seconds() + return self.secs + +def secs_to_hms(secs): + return "%02d:%02d:%06.3f" % (int(secs / 3600), int(secs / 60 % 60), secs % 60) + +def get_slot_val(slot): + return slots[slot]["total"] + +def get_concurrent_val(concurrent): + return concurrency[concurrent]["total"] + +#--------------------------- + +events = [] +for event in sys.stdin: events.append(HistoryEvent(event)) + +if len(events) == 0: + sys.exit(1) + +started = events[0].get_time_secs() +ended = events[-1].get_time_secs() +now = time.time() + +slots = {} +concurrency = {} +active = peak = 0 +for event in events: + slot = slots.get(event.slot, {"active": False, + "start": 0.0, + "total": 0.0}) + + if event.status in ["ACTIVE", "MUTEX"]: + if slot["active"] == False: + active += 1 + concurrent = concurrency.get(active, {"start": 0.0, "total": 0.0}) + concurrent["start"] = event.get_time_secs() + concurrency[active] = concurrent + + slot["active"] = True + slot["start"] = event.get_time_secs() + slots[event.slot] = slot + + if active > peak: + peak = active + + elif slot["active"] == True: + concurrency[active]["total"] += (event.get_time_secs() - concurrency[active]["start"]) + active -= 1 + slot["active"] = False + slot["total"] += (event.get_time_secs() - slot["start"]) + slots[event.slot] = slot + +# If any slots remain active, either the build failed or the build is ongoing, in +# which case "close" the active slots with the current time. +for slotn in slots: + slot = slots[slotn] + if slot["active"] == True: + ended = now + slot["total"] += (now - slot["start"]) + +elapsed = (ended - started) + +print("Total Build Time: %s\n" % secs_to_hms(elapsed)) + +print("Peak concurrency: %d out of %d slots\n" % (peak, len(slots))) + +print("Slot usage (time in ACTIVE state): | Concurrency breakdown:\n") +print("#Rank Slot Usage ( Pct ) # of Slots Usage ( Pct )") + +lines = [] + +for rank, slotn in enumerate(sorted(slots, key=get_slot_val, reverse=True)): + slot = slots[slotn] + pct = (100 * slot["total"] / elapsed) if elapsed > 0.0 else 0.0 + state = " active" if slot["active"] else " " + stime = secs_to_hms(slot["total"]).replace("00:", " :") + lines.append("%s %s (%04.1f%%)%s" % (slotn, stime, pct, state)) + +for rank, concurrentn in enumerate(sorted(concurrency, key=get_concurrent_val, reverse=True)): + concurrent = concurrency[concurrentn] + pct = (100 * concurrent["total"] / elapsed) if elapsed > 0.0 else 0.0 + stime = secs_to_hms(concurrent["total"]).replace("00:", " :") + lines[rank] += " %02d %s (%04.1f%%)" % (concurrentn, stime, pct) + +for rank, line in enumerate(lines): + print(" #%02d %s" % (rank + 1, line)) diff --git a/tools/viewplan b/tools/viewplan new file mode 100755 index 0000000000..3b58af267b --- /dev/null +++ b/tools/viewplan @@ -0,0 +1,29 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST + +. config/options "" +. config/multithread + +# Fake the parallel command if GNU parallel is not available - slow, but works. +fake_parallel() { + while read -r line; do + json_worker "${line}" + done +} + +PARALLEL_BIN=${TOOLCHAIN}/bin/parallel + +[ -x ${PARALLEL_BIN} ] || PARALLEL_BIN=parallel +command -v ${PARALLEL_BIN} >/dev/null || PARALLEL_BIN=fake_parallel + +# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status +set -o pipefail + +cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ + ${PARALLEL_BIN} --plain --no-notice --max-args 30 json_worker --halt now,fail=1 | \ + ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \ + die "FAILURE: Unable to generate plan"