From 0ebc6fef63721708ba7e31d309a9a72c354788aa Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Fri, 8 Feb 2019 17:17:43 +0000 Subject: [PATCH] build system: add parallel build support; use new "image" package --- config/multithread | 140 ++++++++ packages/devel/parallel/package.mk | 11 + packages/virtual/image/package.mk | 44 +++ scripts/autoreconf | 23 +- scripts/build | 35 +- scripts/genbuildplan.py | 380 ++++++++++++++++++++++ scripts/image | 12 +- scripts/image_mt | 498 +++++++++++++++++++++++++++++ scripts/install | 18 +- scripts/unpack | 32 +- tools/dashboard | 19 ++ tools/viewplan | 29 ++ 12 files changed, 1201 insertions(+), 40 deletions(-) create mode 100644 config/multithread create mode 100644 packages/devel/parallel/package.mk create mode 100644 packages/virtual/image/package.mk create mode 100755 scripts/genbuildplan.py create mode 100755 scripts/image_mt create mode 100755 tools/dashboard create mode 100755 tools/viewplan diff --git a/config/multithread b/config/multithread new file mode 100644 index 0000000000..989c4e552b --- /dev/null +++ b/config/multithread @@ -0,0 +1,140 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +THREADCOUNT=${THREADCOUNT:-100%} + +# This function is passed a list of package.mk paths to be processed. +# Each package.mk is sourced with relevant variables output in JSON format. +json_worker() { + local packages="$@" + local pkgpath hierarchy exited + + exit() { exited=1; } + + . config/options "" + + for pkgpath in ${packages}; do + pkgpath="${pkgpath%%@*}" + + exited=0 + if ! source_package "${pkgpath}/package.mk" &>/dev/null; then + unset -f exit + die "$(print_color CLR_ERROR "FAILURE: sourcing package ${pkgpath}/package.mk")" + fi + + [ ${exited} -eq 1 ] && continue + + [[ ${pkgpath} =~ ^${ROOT}/${PACKAGES}/ ]] && hierarchy="global" || hierarchy="local" + + if [ -n "$PKG_ARCH" ]; then + listcontains "$PKG_ARCH" "!$TARGET_ARCH" && continue + listcontains "$PKG_ARCH" "$TARGET_ARCH" || listcontains "$PKG_ARCH" "any" || continue + fi + + cat <" or "install ". +# ${slot} is the job slot number, ie. 1-8 when THREADCOUNT=8. +# ${job} is the sequence within the total number of ${jobs}. +package_worker() { + local slot=$1 job=$2 jobs=$3 args="$4" + local task pkgname result status + + . config/options "" + + [ -f "${THREAD_CONTROL}/pid" ] || echo "${PARALLEL_PID}" >"${THREAD_CONTROL}/pid" + + export MTJOBID=${slot} MTMAXJOBS=${jobs} + + read -r task pkgname <<<$(echo "${args}") + + ${SCRIPTS}/${task} ${pkgname} 2>&1 && result=0 || result=1 + + [ ${result} -eq 0 ] && status="DONE" || status="FAIL" + + ( + flock --exclusive 95 + num=$(($(cat "${THREAD_CONTROL}/progress") + 1)) + cp "${THREAD_CONTROL}/progress" "${THREAD_CONTROL}/progress.prev" + echo "${num}" >"${THREAD_CONTROL}/progress" + printf "[%0*d/%0*d] [%-4s] %-7s %s\n" ${#jobs} ${num} ${#jobs} ${jobs} "${status}" "${task}" "${pkgname}" >&2 + ) 95<"${THREAD_CONTROL}/progress" + + if [ ${result} -eq 0 ]; then + pkg_lock_status "IDLE" + else + pkg_lock_status "FAILED" "${pkgname}" "${task}" + + print_color CLR_ERROR "FAILURE: $SCRIPTS/${task} ${pkgname} has failed!\n" + + if [ -d "${THREAD_CONTROL}/logs" ]; then + cat >&2 <"${THREAD_CONTROL}/progress.prev" + echo 0 >"${THREAD_CONTROL}/progress" + touch "${THREAD_CONTROL}/status" + + [ "${THREADCOUNT}" = "0" ] && THREADCOUNT=1 + + # Bootstrap GNU parallel + $SCRIPTS/build parallel:host 2>&1 || die "Unable to bootstrap parallel package" + + # if number of detected slots is 1 then don't bother using inter-process locks as this is a sequential build + [ $(seq 1 32 | ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-procs ${THREADCOUNT} echo {%} | sort -n | tail -1) -eq 1 ] && withlocks=no || withlocks=yes + + # create a single log file by default if not using locks (single build process), or the builder is a masochist + if [ "${withlocks}" = "no" -a "${ONELOG,,}" != "no" ] || [ "${ONELOG,,}" = "yes" ]; then + logbuffer="--ungroup" + else + mkdir -p "${THREAD_CONTROL}/logs" + logbuffer="--group --results ${THREAD_CONTROL}/logs/{#}/" + fi + + # pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status + set -o pipefail + + cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ + ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \ + ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@} > "${THREAD_CONTROL}"/plan || return + + cat "${THREAD_CONTROL}"/plan | awk '{print $1 " " $2}' | \ + MTBUILDSTART=${now} MTWITHLOCKS=${withlocks} ${TOOLCHAIN}/bin/parallel \ + --plain --no-notice --max-procs ${THREADCOUNT} --joblog="${THREAD_CONTROL}/joblog" ${logbuffer} --halt now,fail=1 --plus \ + package_worker {%} {#} {##} {} || return + + set +o pipefail +} diff --git a/packages/devel/parallel/package.mk b/packages/devel/parallel/package.mk new file mode 100644 index 0000000000..9b62df3999 --- /dev/null +++ b/packages/devel/parallel/package.mk @@ -0,0 +1,11 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +PKG_NAME="parallel" +PKG_VERSION="20190122" +PKG_SHA256="ae735f201a8ceeff2ace48ff779bda9d19846e629bcc02ea7c8768a42394190c" +PKG_LICENSE="GPLv3" +PKG_SITE="https://www.gnu.org/software/parallel/" +PKG_URL="http://ftpmirror.gnu.org/parallel/$PKG_NAME-$PKG_VERSION.tar.bz2" +PKG_DEPENDS_HOST="" +PKG_LONGDESC="GNU parallel is a shell tool for executing jobs in parallel using one or more computers." diff --git a/packages/virtual/image/package.mk b/packages/virtual/image/package.mk new file mode 100644 index 0000000000..12ecfaacfe --- /dev/null +++ b/packages/virtual/image/package.mk @@ -0,0 +1,44 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +PKG_NAME="image" +PKG_LICENSE="GPL" +PKG_SITE="https://libreelec.tv" +PKG_DEPENDS_TARGET="toolchain squashfs:host dosfstools:host fakeroot:host kmod:host mtools:host populatefs:host libc gcc linux linux-drivers linux-firmware ${BOOTLOADER} busybox util-linux corefonts network misc-packages debug" +PKG_SECTION="virtual" +PKG_LONGDESC="Root package used to build and create complete image" + +# Graphic support +[ ! "$DISPLAYSERVER" = "no" ] && PKG_DEPENDS_TARGET+=" $DISPLAYSERVER" + +# Multimedia support +[ ! "$MEDIACENTER" = "no" ] && PKG_DEPENDS_TARGET+=" mediacenter" + +# Sound support +[ "$ALSA_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" alsa" + +# Automounter support +[ "$UDEVIL" = "yes" ] && PKG_DEPENDS_TARGET+=" udevil" + +# EXFAT support +[ "$EXFAT" = "yes" ] && PKG_DEPENDS_TARGET+=" fuse-exfat" + +# NTFS 3G support +[ "$NTFS3G" = "yes" ] && PKG_DEPENDS_TARGET+=" ntfs-3g_ntfsprogs" + +# Remote support +[ "$REMOTE_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" remote" + +# Virtual image creation support +[ "$PROJECT" = "Generic" ] && PKG_DEPENDS_TARGET+=" virtual" + +# Installer support +[ "$INSTALLER_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" installer" + +# Devtools... (not for Release) +[ "$TESTING" = "yes" ] && PKG_DEPENDS_TARGET+=" testing" + +# OEM packages +[ "$OEM_SUPPORT" = "yes" ] && PKG_DEPENDS_TARGET+=" oem" + +true diff --git a/scripts/autoreconf b/scripts/autoreconf index 1ad473fe38..26cc2ca1bc 100755 --- a/scripts/autoreconf +++ b/scripts/autoreconf @@ -12,11 +12,24 @@ if [ ! -f "${PKG_BUILD}/configure.in" \ fi if [ ! -f "${PKG_BUILD}/.autoreconf-done" ]; then - touch "${PKG_BUILD}/NEWS" "${PKG_BUILD}/AUTHORS" "${PKG_BUILD}/ChangeLog" - mkdir -p "${PKG_BUILD}/m4" + PARENT_PKG="${2}" - build_msg "CLR_AUTORECONF" "AUTORECONF" "${1}" "indent" + # lock package during autoreconf otherwise it is racy, eg. glib:host/glib:target building concurrently + pkg_lock "${PKG_NAME}" "reconf" "${PARENT_PKG}" - do_autoreconf "${PKG_BUILD}" - touch "${PKG_BUILD}/.autoreconf-done" + if [ ! -f "${PKG_BUILD}/.autoreconf-done" ]; then + pkg_lock_status "ACTIVE" "${PKG_NAME}" "reconf" + + touch "${PKG_BUILD}/NEWS" "${PKG_BUILD}/AUTHORS" "${PKG_BUILD}/ChangeLog" + mkdir -p "${PKG_BUILD}/m4" + + build_msg "CLR_AUTORECONF" "AUTORECONF" "${PKG_NAME}" "indent" + + do_autoreconf "${PKG_BUILD}" + touch "${PKG_BUILD}/.autoreconf-done" + + pkg_lock_status "UNLOCK" "${PKG_NAME}" "reconf" "configured" + else + pkg_lock_status "UNLOCK" "${PKG_NAME}" "reconf" "already configured" + fi fi diff --git a/scripts/build b/scripts/build index 1746d8795b..46a9e77b7c 100755 --- a/scripts/build +++ b/scripts/build @@ -7,7 +7,7 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name[:]" + die "usage: $0 package_name[:] [parent_pkg]" fi if [ "$1" = "--all" ]; then @@ -32,10 +32,13 @@ if [ "${1//:/}" != "${1}" ]; then PACKAGE_NAME="${1%:*}" TARGET="${1#*:}" else - PACKAGE_NAME=$1 + PACKAGE_NAME="${1}" TARGET= fi TARGET="${TARGET:-target}" +PARENT_PKG="${2:-${PKG_NAME}:${TARGET}}" + +pkg_lock "${PACKAGE_NAME}:${TARGET}" "build" "${PARENT_PKG}" mkdir -p $STAMPS/$PACKAGE_NAME STAMP=$STAMPS/$PACKAGE_NAME/build_$TARGET @@ -46,10 +49,11 @@ if [ -f $STAMP ]; then rm -f $STAMP elif [ ! "$BUILD_WITH_DEBUG" = "$STAMP_BUILD_WITH_DEBUG" ]; then rm -f $STAMP - elif [ "$1" = "u-boot" -a ! "$UBOOT_SYSTEM" = "$STAMP_UBOOT_SYSTEM" ]; then + elif [ "${PKG_NAME}" = "u-boot" -a ! "$UBOOT_SYSTEM" = "$STAMP_UBOOT_SYSTEM" ]; then rm -f $STAMP else # stamp matched: already built, do nothing + pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "already built" exit 0 fi fi @@ -65,16 +69,9 @@ case "$TARGET" in "bootstrap") _pkg_depends="$PKG_DEPENDS_BOOTSTRAP";; esac for p in $_pkg_depends; do - $SCRIPTS/build $p + $SCRIPTS/build "${p}" "${PARENT_PKG}" done -# build this package -if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then - build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})") [DEBUG]" "indent" -else - build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})")" "indent" -fi - # virtual packages are not built as they only contain dependencies, so dont go further here if [ "$PKG_SECTION" = "virtual" ]; then PKG_DEEPHASH=$(calculate_stamp) @@ -82,9 +79,17 @@ if [ "$PKG_SECTION" = "virtual" ]; then echo "STAMP_$i=\"${!i}\"" >> $STAMP done + pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "built" exit 0 fi +# build this package +if [ "${BUILD_WITH_DEBUG}" = "yes" ]; then + build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})") [DEBUG]" "indent" +else + build_msg "CLR_BUILD" "BUILD" "${PACKAGE_NAME} $(print_color "CLR_TARGET" "(${TARGET})")" "indent" +fi + setup_toolchain $TARGET # configure install directory @@ -201,9 +206,11 @@ build_msg "CLR_TOOLCHAIN" "TOOLCHAIN" "${PKG_TOOLCHAIN}${_auto_toolchain}" # make autoreconf if [ "$PKG_TOOLCHAIN" = "autotools" ]; then - $SCRIPTS/autoreconf $PACKAGE_NAME + $SCRIPTS/autoreconf "${PACKAGE_NAME}" "${PARENT_PKG}" fi +pkg_lock_status "ACTIVE" "${PKG_NAME}:${TARGET}" "build" + # include build template and build pkg_call_exists pre_build_$TARGET && pkg_call pre_build_$TARGET @@ -455,6 +462,8 @@ PKG_DEEPHASH=$(calculate_stamp) for i in PKG_NAME PKG_DEEPHASH BUILD_WITH_DEBUG; do echo "STAMP_$i=\"${!i}\"" >> $STAMP done -if [ "$1" = "u-boot" ]; then +if [ "${PKG_NAME}" = "u-boot" ]; then echo "STAMP_UBOOT_SYSTEM=\"${UBOOT_SYSTEM}\"" >> $STAMP fi + +pkg_lock_status "UNLOCK" "${PKG_NAME}:${TARGET}" "build" "built" diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py new file mode 100755 index 0000000000..0a3a078c56 --- /dev/null +++ b/scripts/genbuildplan.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +from __future__ import print_function +import sys, os, codecs, json, argparse, re + +ROOT_PKG = "__root__" + +class LibreELEC_Package: + def __init__(self, name, section): + self.name = name + self.section = section + self.deps = {"bootstrap": [], + "init": [], + "host": [], + "target": []} + self.wants = [] + self.wantedby = [] + + def __repr__(self): + s = "%-9s: %s" % ("name", self.name) + s = "%s\n%-9s: %s" % (s, "section", self.section) + + for t in self.deps: + s = "%s\n%-9s: %s" % (s, t, self.deps[t]) + + s = "%s\n%-9s: %s" % (s, "NEEDS", self.wants) + s = "%s\n%-9s: %s" % (s, "WANTED BY", self.wantedby) + + return s + + def addDependencies(self, target, packages): + for d in " ".join(packages.split()).split(): + self.deps[target].append(d) + name = d.split(":")[0] + if name not in self.wants and name != self.name: + self.wants.append(name) + + def delDependency(self, target, package): + if package in self.deps[target]: + self.deps[target].remove(package) + name = package.split(":")[0] + if name in self.wants: + self.wants.remove(name) + + def addReference(self, package): + name = package.split(":")[0] + if name not in self.wantedby: + self.wantedby.append(name) + + def delReference(self, package): + name = package.split(":")[0] + if name in self.wantedby: + self.wantedby.remove(name) + + def isReferenced(self): + return False if self.wants == [] else True + + def isWanted(self): + return False if self.wantedby == [] else True + + def references(self, package): + return package in self.wants + +# Reference material: +# https://www.electricmonk.nl/docs/dependency_resolving_algorithm/dependency_resolving_algorithm.html +class Node: + def __init__(self, name, target, section): + self.name = name + self.target = target + self.section = section + self.fqname = "%s:%s" % (name, target) + self.edges = [] + + def appendEdges(self, node): + # Add the node itself... + if node not in self.edges: + self.edges.append(node) + # as well as its edges + for e in node.edges: + if e not in self.edges: + self.edges.append(e) + + # Return True if the dependencies of the specified node are met by this node + def satisfies(self, node): + for e in node.edges: + if e not in self.edges: + return False + return True + + def __repr__(self): + s = "%-9s: %s" % ("name", self.name) + s = "%s\n%-9s: %s" % (s, "target", self.target) + s = "%s\n%-9s: %s" % (s, "fqname", self.fqname) + s = "%s\n%-9s: %s" % (s, "common", self.commonName()) + s = "%s\n%-9s: %s" % (s, "section", self.section) + + for e in self.edges: + s = "%s\nEDGE: %s" % (s, e.fqname) + + return s + + def commonName(self): + return self.name if self.target == "target" else "%s:%s" % (self.name, self.target) + + def addEdge(self, node): + self.edges.append(node) + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +# Read a JSON list of all possible packages from stdin +def loadPackages(): + jdata = json.loads("[%s]" % sys.stdin.read().replace('\n','')[:-1]) + + map = {} + + # Load "global" packages first + for pkg in jdata: + if pkg["hierarchy"] == "global": + map[pkg["name"]] = initPackage(pkg) + + # Then the "local" packages, as these will replace any matching "global" packages + for pkg in jdata: + if pkg["hierarchy"] == "local": + map[pkg["name"]] = initPackage(pkg) + + return map + +# Create a fully formed LibreELEC_Package object +def initPackage(package): + pkg = LibreELEC_Package(package["name"], package["section"]) + + for target in ["bootstrap", "init", "host", "target"]: + pkg.addDependencies(target, package[target]) + + return pkg + +# Split name:target into components +def split_package(name): + parts = name.split(":") + pn = parts[0] + pt = parts[1] if len(parts) != 1 else "target" + return (pn, pt) + +# Return a list of packages of the specified type +def get_packages_by_target(target, list): + newlist = [] + + for p in list: + (pn, pt) = split_package(p) + if target in ["target", "init"] and pt in ["target", "init"]: + newlist.append(p) + elif target in ["bootstrap", "host"] and pt in ["bootstrap", "host"]: + newlist.append(p) + + return newlist + +# For the specified node iterate over the list of scheduled nodes and return the first +# position where we could possibly build this node (ie. all dependencies satisfied). +def findbuildpos(node, list): + + # Keep a running total of all dependencies as we progress through the list + alldeps = Node("", "", "") + + candidate = None + for n in list: + alldeps.appendEdges(n) + if alldeps.satisfies(node): + if len(n.edges) > len(node.edges): + if candidate == None: + candidate = n + break + candidate = n + + return list.index(candidate) + 1 if candidate else -1 + +# Resolve dependencies for a node +def dep_resolve(node, resolved, unresolved, noreorder): + unresolved.append(node) + + for edge in node.edges: + if edge not in resolved: + if edge in unresolved: + raise Exception('Circular reference detected: %s -> %s\nRemove %s from %s package.mk::PKG_DEPENDS_%s' % \ + (node.fqname, edge.commonName(), edge.commonName(), node.name, node.target.upper())) + dep_resolve(edge, resolved, unresolved, noreorder) + + if node not in resolved: + pos = -1 if noreorder else findbuildpos(node, resolved) + if pos != -1: + resolved.insert(pos, node) + else: + resolved.append(node) + + unresolved.remove(node) + +# Return a list of build steps for the trigger packages +def get_build_steps(args, nodes, trigger_pkgs, built_pkgs): + resolved = [] + unresolved = [] + + for pkgname in [x for x in trigger_pkgs if x]: + if pkgname.find(":") == -1: + pkgname = "%s:target" % pkgname + + if pkgname in nodes: + dep_resolve(nodes[pkgname], resolved, unresolved, args.no_reorder) + + # Abort if any references remain unresolved + if unresolved != []: + eprint("The following dependencies have not been resolved:") + for dep in unresolved: + eprint(" %s" % dep) + raise("Unresolved references") + + # Output list of resolved dependencies + for pkg in resolved: + if pkg.fqname not in built_pkgs: + built_pkgs.append(pkg.fqname) + task = "build" if pkg.fqname.endswith(":host") else "install" + yield(task, pkg.fqname) + +# Reduce the complete list of packages to a map of those packages that will +# be needed for the build. +def processPackages(args, packages, build): + # Add dummy package to ensure build/install dependencies are not culled + pkg = { + "name": ROOT_PKG, + "section": "virtual", + "hierarchy": "global", + "bootstrap": "", + "init": "", + "host": " ".join(get_packages_by_target("host", build)), + "target": " ".join(get_packages_by_target("target", build)) + } + + packages[pkg["name"]] = initPackage(pkg) + + # Resolve reverse references that we can use to ignore unreferenced packages + for pkgname in packages: + for opkgname in packages: + opkg = packages[opkgname] + if opkg.references(pkgname): + if pkgname in packages: + packages[pkgname].addReference(opkgname) + + # Identify unused packages + while True: + changed = False + for pkgname in packages: + pkg = packages[pkgname] + if pkg.isWanted(): + for opkgname in pkg.wantedby: + if opkgname != ROOT_PKG: + if not packages[opkgname].isWanted(): + pkg.delReference(opkgname) + changed = True + if not changed: + break + + # Create a new map of "needed" packages + needed_map = {} + for pkgname in packages: + pkg = packages[pkgname] + if pkg.isWanted() or pkgname == ROOT_PKG: + needed_map[pkgname] = pkg + + # Validate package dependency references + for pkgname in needed_map: + pkg = needed_map[pkgname] + for t in pkg.deps: + for d in pkg.deps[t]: + if split_package(d)[0] not in needed_map and not args.ignore_invalid: + msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper()) + if args.warn_invalid: + eprint("WARNING: %s" % msg) + else: + raise Exception(msg) + + node_map = {} + + # Convert all packages to target-specific nodes + for pkgname in needed_map: + pkg = needed_map[pkgname] + for target in pkg.deps: + if pkg.deps[target]: + node = Node(pkgname, target, pkg.section) + node_map[node.fqname] = node + + # Ensure all referenced dependencies exist as a basic node + for pkgname in needed_map: + pkg = needed_map[pkgname] + for target in pkg.deps: + for dep in pkg.deps[target]: + dfq = dep if dep.find(":") != -1 else "%s:target" % dep + if dfq not in node_map: + (dfq_p, dfq_t) = split_package(dfq) + if dfq_p in packages: + dpkg = packages[dfq_p] + node_map[dfq] = Node(dfq_p, dfq_t, dpkg.section) + elif not args.ignore_invalid: + raise Exception("Invalid package! Package %s cannot be found for this PROJECT/DEVICE/ARCH" % dfq_p) + + # To each target-specific node, add the correspnding + # target-specific dependency nodes ("edges") + for name in node_map: + node = node_map[name] + if node.name not in needed_map: + if args.warn_invalid: + continue + else: + raise Exception("Invalid package! Package %s cannot be found for this PROJECT/DEVICE/ARCH" % node.name) + for dep in needed_map[node.name].deps[node.target]: + dfq = dep if dep.find(":") != -1 else "%s:target" % dep + if dfq in node_map: + node.addEdge(node_map[dfq]) + + return node_map + +#--------------------------------------------- +parser = argparse.ArgumentParser(description="Generate package dependency list for the requested build/install packages. \ + Package data will be read from stdin in JSON format.", \ + formatter_class=lambda prog: argparse.HelpFormatter(prog,max_help_position=25,width=90)) + +parser.add_argument("-b", "--build", nargs="+", metavar="PACKAGE", required=True, \ + help="Space-separated list of build trigger packages, either for host or target. Required property - specify at least one package.") + +parser.add_argument("--warn-invalid", action="store_true", \ + help="Warn about invalid/missing dependency packages, perhaps excluded by a PKG_ARCH incompatability. Default is to abort.") + +parser.add_argument("--no-reorder", action="store_true", default="True", \ + help="Do not resequence steps based on dependencies. This is the default.") + +parser.add_argument("--reorder", action="store_false", dest="no_reorder", \ + help="Disable --no-reorder and resequence packages to try and reduce stalls etc.") + +parser.add_argument("--show-wants", action="store_true", \ + help="Output \"wants\" dependencies for each step.") + +parser.add_argument("--hide-wants", action="store_false", dest="show_wants", default="True", \ + help="Disable --show-wants.") + +parser.add_argument("--ignore-invalid", action="store_true", \ + help="Ignore invalid packages.") + +args = parser.parse_args() + +ALL_PACKAGES = loadPackages() + +loaded = len(ALL_PACKAGES) + +REQUIRED_PKGS = processPackages(args, ALL_PACKAGES, args.build) + +# Output list of packages to build/install +built_pkgs = [] +steps = [] + +for step in get_build_steps(args, REQUIRED_PKGS, args.build, built_pkgs): + steps.append(step) + +eprint("Packages loaded : %d" % loaded) +eprint("Build trigger(s): %d [%s]" % (len(args.build), " ".join(args.build))) +eprint("Package steps : %d" % len(steps)) +eprint("") + +# Output build/install steps +if args.show_wants: + for step in steps: + wants = [] + node = (REQUIRED_PKGS[step[1]]) + for e in node.edges: + wants.append(e.fqname) + print("%-7s %-25s (wants: %s)" % (step[0], step[1].replace(":target",""), ", ".join(wants).replace(":target",""))) +else: + for step in steps: + print("%-7s %s" % (step[0], step[1].replace(":target",""))) diff --git a/scripts/image b/scripts/image index 57febd43b6..bd4decf390 100755 --- a/scripts/image +++ b/scripts/image @@ -97,14 +97,14 @@ mkdir -p $INSTALL for directory in etc dev proc run sys tmp usr var flash storage; do mkdir -p $INSTALL/$directory done -ln -sf /var/media $INSTALL/media -ln -sf /usr/lib $INSTALL/lib -ln -sf /usr/bin $INSTALL/bin -ln -sf /usr/sbin $INSTALL/sbin +ln -sfn /var/media $INSTALL/media +ln -sfn /usr/lib $INSTALL/lib +ln -sfn /usr/bin $INSTALL/bin +ln -sfn /usr/sbin $INSTALL/sbin if [ "$TARGET_ARCH" = "x86_64" ]; then - ln -s /usr/lib $INSTALL/lib64 - ln -s /usr/lib $INSTALL/usr/lib64 + ln -sfn /usr/lib $INSTALL/lib64 + ln -sfn /usr/lib $INSTALL/usr/lib64 fi echo "$TARGET_VERSION" > $INSTALL/etc/release diff --git a/scripts/image_mt b/scripts/image_mt new file mode 100755 index 0000000000..0bf4334d1f --- /dev/null +++ b/scripts/image_mt @@ -0,0 +1,498 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST + +. config/options "" +. config/multithread +. config/show_config + +show_config +save_build_config + +# Setup both toolchain cmake configs to avoid potentially racy behaviour later. +# Use a fork for host to isolate any variable modifications. +( setup_toolchain host ) +setup_toolchain target + +if [ -n "$CUSTOM_GIT_HASH" ]; then + GIT_HASH="$CUSTOM_GIT_HASH" +else + GIT_HASH=$(git rev-parse HEAD) +fi + +if [ "$LIBREELEC_VERSION" = "devel" ]; then + GIT_ABBREV=${GIT_HASH:0:7} + DEVEL_VERSION=$LIBREELEC_VERSION + case "$BUILD_PERIODIC" in + nightly) LIBREELEC_VERSION=nightly-$(date +%Y%m%d)-$GIT_ABBREV;; + daily) LIBREELEC_VERSION=daily-$(date +%Y%j)-$GIT_ABBREV;; + weekly) LIBREELEC_VERSION=weekly-$(date +%G%V)-$GIT_ABBREV;; + monthly) LIBREELEC_VERSION=monthly-$(date +%Y%m)-$GIT_ABBREV;; + *) LIBREELEC_VERSION=devel-$(date +%Y%m%d%H%M%S)-$GIT_ABBREV;; + esac +fi + +# Get origin url, fix git:// and git@github.com: urls if necessary +ORIGIN_URL="$(git remote -v | awk '$1 == "origin" { print $2 }' | head -1 | sed 's#\.git$##;s#^git:#https:#;s#^git@github\.com:#https://github.com/#')" + +[ "${BUILDER_NAME,,}" = "official" ] && BUILDER_NAME= +if [ "$OFFICIAL" = "yes" ]; then + LIBREELEC_BUILD="official" +else + if [ -n "$BUILDER_NAME" ]; then + LIBREELEC_BUILD="$BUILDER_NAME" + else + LIBREELEC_BUILD="community" + fi +fi + +if [ -n "$CUSTOM_VERSION" ]; then + LIBREELEC_VERSION="$CUSTOM_VERSION" +fi + +LIBREELEC_ARCH="${DEVICE:-$PROJECT}.$TARGET_ARCH" +TARGET_VERSION="$LIBREELEC_ARCH-$LIBREELEC_VERSION" + +if [ -n "$CUSTOM_IMAGE_NAME" ]; then + IMAGE_NAME="$CUSTOM_IMAGE_NAME" +else + if [ "$DEVEL_VERSION" = "devel" ]; then + IMAGE_NAME="$DISTRONAME-$LIBREELEC_ARCH-$OS_VERSION-$LIBREELEC_VERSION" + else + IMAGE_NAME="$DISTRONAME-$TARGET_VERSION" + fi + + if [ -n "$UBOOT_SYSTEM" ] && [ "$UBOOT_SYSTEM" != "${DEVICE:-$PROJECT}" ]; then + IMAGE_NAME="$IMAGE_NAME-$UBOOT_SYSTEM" + fi +fi + +if [ -n "$IMAGE_SUFFIX" ]; then + IMAGE_NAME="$IMAGE_NAME-$IMAGE_SUFFIX" +fi + +echo "$IMAGE_NAME" > $BUILD/BUILD_FILENAME + +# Setup fakeroot +rm -rf $FAKEROOT_SCRIPT # remove $FAKEROOT_SCRIPT if it exist +touch $FAKEROOT_SCRIPT # create an empty $FAKEROOT_SCRIPT +chmod +x $FAKEROOT_SCRIPT # make $FAKEROOT_SCRIPT executable +echo "chown -R 0:0 $INSTALL" >> $FAKEROOT_SCRIPT + +# Clean old install dirs +rm -rf $INSTALL +rm -rf $STAMPS_INSTALL +mkdir -p $INSTALL + +# Create base layout of LibreELEC read-only file system +for directory in etc dev proc run sys tmp usr var flash storage; do + mkdir -p $INSTALL/$directory +done + +# Build image contents +start_multithread_build image || die "Parallel build failure - see log for details. Time of failure: $(date)" + +echo "Successful build, creating image..." >&2 + +# Create legacy sym links +ln -sfn /var/media $INSTALL/media +ln -sfn /usr/lib $INSTALL/lib +ln -sfn /usr/bin $INSTALL/bin +ln -sfn /usr/sbin $INSTALL/sbin + +if [ "$TARGET_ARCH" = "x86_64" ]; then + ln -sfn /usr/lib $INSTALL/lib64 + ln -sfn /usr/lib $INSTALL/usr/lib64 +fi + +echo "$TARGET_VERSION" > $INSTALL/etc/release + +# Create /etc/os-release +echo -e "NAME=\"$DISTRONAME\"" > $INSTALL/etc/os-release +echo -e "VERSION=\"$LIBREELEC_VERSION\"" >> $INSTALL/etc/os-release +echo -e "ID=\"libreelec\"" >> $INSTALL/etc/os-release +echo -e "VERSION_ID=\"$OS_VERSION\"" >> $INSTALL/etc/os-release +echo -e "PRETTY_NAME=\"$DISTRONAME ($LIBREELEC_BUILD): $LIBREELEC_VERSION\"" >> $INSTALL/etc/os-release +echo -e "HOME_URL=\"https://libreelec.tv\"" >> $INSTALL/etc/os-release +echo -e "BUG_REPORT_URL=\"$ORIGIN_URL\"" >> $INSTALL/etc/os-release +echo -e "BUILD_ID=\"$GIT_HASH\"" >> $INSTALL/etc/os-release +echo -e "OPENELEC_ARCH=\"$LIBREELEC_ARCH\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_ARCH=\"$LIBREELEC_ARCH\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_BUILD=\"$LIBREELEC_BUILD\"" >> $INSTALL/etc/os-release +echo -e "LIBREELEC_PROJECT=\"$PROJECT\"" >> $INSTALL/etc/os-release +[ -n "$DEVICE" ] && echo -e "LIBREELEC_DEVICE=\"$DEVICE\"" >> $INSTALL/etc/os-release +[ -n "$BUILDER_NAME" ] && echo -e "BUILDER_NAME=\"$BUILDER_NAME\"" >> $INSTALL/etc/os-release +[ -n "$BUILDER_VERSION" ] && echo -e "BUILDER_VERSION=\"$BUILDER_VERSION\"" >> $INSTALL/etc/os-release + +# Create /etc/issue +echo "$GREETING0" > $INSTALL/etc/issue +echo "$GREETING1" >> $INSTALL/etc/issue +echo "$GREETING2" >> $INSTALL/etc/issue +echo "$GREETING3" >> $INSTALL/etc/issue +echo "$GREETING4" >> $INSTALL/etc/issue +echo "$DISTRONAME ($LIBREELEC_BUILD): $LIBREELEC_VERSION ($LIBREELEC_ARCH)" >> $INSTALL/etc/issue + +ln -sf /etc/issue $INSTALL/etc/motd + +# Copy PROJECT related files to filesystem +if [ -d "$PROJECT_DIR/$PROJECT/filesystem" ]; then + cp -PR $PROJECT_DIR/$PROJECT/filesystem/* $INSTALL + # Install project specific systemd services + for service in $PROJECT_DIR/$PROJECT/filesystem/usr/lib/systemd/system/*.service; do + if [ -f "$service" ]; then + enable_service $(basename $service) + fi + done +fi + +# Copy DEVICE related files to filesystem +if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem" ]; then + cp -PR $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/* $INSTALL + # Install device specific systemd services + for service in $PROJECT_DIR/$PROJECT/devices/$DEVICE/filesystem/usr/lib/systemd/system/*.service; do + if [ -f "$service" ]; then + enable_service $(basename $service) + fi + done +fi + +# Run depmod for base overlay modules +MODVER=$(basename $(ls -d $INSTALL/usr/lib/kernel-overlays/base/lib/modules/*)) +find $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/ -name *.ko | \ + sed -e "s,$INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/,," \ + > $INSTALL/usr/lib/kernel-overlays/base/lib/modules/$MODVER/modules.order +$TOOLCHAIN/bin/depmod -b $INSTALL/usr/lib/kernel-overlays/base -a -e -F "$BUILD/linux-$(kernel_version)/System.map" $MODVER 2>&1 + +# Strip kernel modules +for MOD in $(find $INSTALL/usr/lib/kernel-overlays/ -type f -name *.ko); do + ${TARGET_KERNEL_PREFIX}strip --strip-debug $MOD +done + +# Symlink overlayed modules to /usr/lib/modules +ln -sT /var/lib/modules $INSTALL/usr/lib/modules + +# Symlink overlayed firmware to /usr/lib/firmware +ln -sT /var/lib/firmware $INSTALL/usr/lib/firmware + +# Make target dir +mkdir -p $TARGET_IMG +rm -rf $TARGET_IMG/$IMAGE_NAME.kernel + +# Copy kernel to target dir +cp -PR $BUILD/linux-$(kernel_version)/arch/$TARGET_KERNEL_ARCH/boot/$KERNEL_TARGET $TARGET_IMG/$IMAGE_NAME.kernel +chmod 0644 $TARGET_IMG/$IMAGE_NAME.kernel + +# Set mksquashfs options for each compression method +if [ -z "$SQUASHFS_COMPRESSION_OPTION" ]; then + if [ "${SQUASHFS_COMPRESSION:-gzip}" = "gzip" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 9 -b 262144" + elif [ "$SQUASHFS_COMPRESSION" = "lzo" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 9 -b 524288" + elif [ "$SQUASHFS_COMPRESSION" = "zstd" ]; then + SQUASHFS_COMPRESSION_OPTION="-Xcompression-level 22 -b 262144" + fi +fi + +# Create squashfs file, default to gzip if no compression configured +echo "rm -rf \"$TARGET_IMG/$IMAGE_NAME.system\"" >> $FAKEROOT_SCRIPT +echo "$TOOLCHAIN/bin/mksquashfs \"$BUILD/image/system\" \"$TARGET_IMG/$IMAGE_NAME.system\" -noappend -comp ${SQUASHFS_COMPRESSION:-gzip} ${SQUASHFS_COMPRESSION_OPTION}" >> $FAKEROOT_SCRIPT + +# Run fakeroot +$TOOLCHAIN/bin/fakeroot -- $FAKEROOT_SCRIPT +rm -rf $FAKEROOT_SCRIPT + +# Set permissions +chmod 0644 $TARGET_IMG/$IMAGE_NAME.system + +if [ "$1" = "release" -o "$1" = "mkimage" -o "$1" = "amlpkg" -o "$1" = "noobs" ]; then + + RELEASE_DIR="target/$IMAGE_NAME" + + # Cleanup + rm -rf $RELEASE_DIR + + # Create release dir + mkdir -p $RELEASE_DIR + + # Remove any previously created release images + rm -rf $TARGET_IMG/$IMAGE_NAME.img.gz + + if [ -n "$BOOTLOADER" ]; then + + BOOTLOADER_DIR="$(get_pkg_directory "$BOOTLOADER")" + + if [ -d $BOOTLOADER_DIR/files ]; then + cp -R $BOOTLOADER_DIR/files/* $RELEASE_DIR + fi + + if find_file_path bootloader/release $BOOTLOADER_DIR/release; then + echo "Running $FOUND_PATH" + . $FOUND_PATH + fi + fi + + cp $ROOT/README* $RELEASE_DIR + cp $ROOT/CHANGELOG* $RELEASE_DIR + echo "$TARGET_VERSION" > $RELEASE_DIR/RELEASE + + if [ ! "$MEDIACENTER" = "no" ]; then + echo "Kodi commit: $(get_pkg_version $MEDIACENTER)" >> $RELEASE_DIR/RELEASE + fi + + mkdir -p $RELEASE_DIR/licenses + cp $ROOT/licenses/* $RELEASE_DIR/licenses + + mkdir -p $RELEASE_DIR/target + cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/target/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/target/KERNEL + + # Create md5sum's + ( cd $RELEASE_DIR; + md5sum -t target/SYSTEM > target/SYSTEM.md5; + md5sum -t target/KERNEL > target/KERNEL.md5; + ) + + # Create target directory + mkdir -p $TARGET_IMG + + # Remove any previously created release tarballs + rm -rf $TARGET_IMG/$IMAGE_NAME.tar + + # Create release tarball + tar cf $TARGET_IMG/$IMAGE_NAME.tar -C target $IMAGE_NAME + + # Create sha256 checksum of tarball + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}.tar > ${IMAGE_NAME}.tar.sha256 + ) + + # Create image files if requested + if [[ ( "$1" = "amlpkg" || "$1" = "noobs" || "$1" = "mkimage" ) && -n "$BOOTLOADER" ]]; then + # INSTALL_SRC_DIR can be board specific + if [ -n "$DEVICE" -a -d "$PROJECT_DIR/$PROJECT/devices/$DEVICE/install" ]; then + INSTALL_SRC_DIR="$PROJECT_DIR/$PROJECT/devices/$DEVICE/install" + else + INSTALL_SRC_DIR="$PROJECT_DIR/$PROJECT/install" + fi + + # Variables used in image script must be passed + env \ + PATH="$PATH:/sbin" \ + ROOT="$ROOT" \ + SCRIPTS="$SCRIPTS" \ + TOOLCHAIN="$TOOLCHAIN" \ + PROJECT_DIR="$PROJECT_DIR" \ + PROJECT="$PROJECT" \ + DEVICE="$DEVICE" \ + DISTRO="$DISTRO" \ + TARGET_IMG="$TARGET_IMG" \ + IMAGE_NAME="$IMAGE_NAME" \ + INSTALL_SRC_DIR="$INSTALL_SRC_DIR" \ + BOOTLOADER="$BOOTLOADER" \ + KERNEL_NAME="$KERNEL_NAME" \ + TARGET_KERNEL_ARCH="$TARGET_KERNEL_ARCH" \ + RELEASE_DIR=$RELEASE_DIR \ + UUID_STORAGE="$(uuidgen)" \ + DISTRO_BOOTLABEL="$DISTRO_BOOTLABEL" \ + DISTRO_DISKLABEL="$DISTRO_DISKLABEL" \ + UBOOT_SYSTEM="$UBOOT_SYSTEM" \ + UBOOT_VERSION="$UBOOT_VERSION" \ + EXTRA_CMDLINE="$EXTRA_CMDLINE" \ + SYSTEM_SIZE="$SYSTEM_SIZE" \ + SYSTEM_PART_START="$SYSTEM_PART_START" \ + OVA_SIZE="$OVA_SIZE" \ + $SCRIPTS/mkimage + fi + + # Cleanup release dir + rm -rf $RELEASE_DIR + + # Create WeTek Play (Amlogic) ZIP update and auto-install packages if requested + if [ "$1" = "amlpkg" ]; then + echo "Creating Amlogic ZIP update package" + + AML_PKG_DIR="$RELEASE_DIR/ampl-pkg" + + # Create package directory + mkdir -p "$AML_PKG_DIR" + + # Copy system and kernel images + mkdir -p "$AML_PKG_DIR/system" + cp $TARGET_IMG/$IMAGE_NAME.system $AML_PKG_DIR/system/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $AML_PKG_DIR/KERNEL + + # Copy update-binary and updater-script + META_INF_DIR="$AML_PKG_DIR/META-INF/com/google/android" + mkdir -p "$META_INF_DIR" + cp $INSTALL_SRC_DIR/update-binary $META_INF_DIR + cp $INSTALL_SRC_DIR/updater-script $META_INF_DIR + + # Copy other files if any + if [ -d "$INSTALL_SRC_DIR/files" ]; then + cp -PR $INSTALL_SRC_DIR/files/* $AML_PKG_DIR + fi + + # Copy device tree image if any + if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then + cp "$INSTALL/usr/share/bootloader/dtb.img" $AML_PKG_DIR/dtb.img + fi + + # Create the update package + pushd "$AML_PKG_DIR" > /dev/null + zip -rq update.zip * + + # Sign the update package + echo "Signing the update package" + mkdir -p sign + SIGNAPK_DIR="$ROOT/tools/signapk" + java -Xmx1024m -jar $SIGNAPK_DIR/signapk.jar -w $SIGNAPK_DIR/testkey.x509.pem $SIGNAPK_DIR/testkey.pk8 update.zip sign/$IMAGE_NAME-update.zip + + # Create the auto-install package + echo "Creating Amlogic ZIP auto-install package" + pushd sign > /dev/null + echo --update_package=/sdcard/$IMAGE_NAME-update.zip > factory_update_param.aml + echo --wipe_data >> factory_update_param.aml + echo --wipe_cache >> factory_update_param.aml + if [ -f "$INSTALL_SRC_DIR/files/recovery.img" ]; then + cp $INSTALL_SRC_DIR/files/recovery.img . + fi + + if [ -f $INSTALL_SRC_DIR/files/aml_autoscript ]; then + cp $INSTALL_SRC_DIR/files/aml_autoscript . + fi + + # Copy device tree image if any + if [ -f "$INSTALL/usr/share/bootloader/dtb.img" ]; then + cp "$INSTALL/usr/share/bootloader/dtb.img" . + fi + + zip -q $TARGET_IMG/$IMAGE_NAME.zip * + + # Create sha256 checksum of zip + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}.zip > ${IMAGE_NAME}.zip.sha256 + ) + + popd > /dev/null + popd > /dev/null + + elif [ "$1" = "noobs" ]; then + echo "Creating \"$1\" release tarball..." + + RELEASE_DIR="$TARGET_IMG/${IMAGE_NAME}-$1" + + # eg. LibreELEC_RPi, LibreELEC_RPi2 etc. + NOOBS_DISTRO="${DISTRONAME}_${DEVICE:-$PROJECT}" + + # Create release dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO} + + if [ -f $DISTRO_DIR/$DISTRO/${DISTRONAME}_40x40.png ]; then + cp -PR $DISTRO_DIR/$DISTRO/${DISTRONAME}_40x40.png $RELEASE_DIR/${NOOBS_DISTRO}/${NOOBS_DISTRO}.png + else + cp -PR $DISTRO_DIR/$DISTRO/${DISTRONAME}.png $RELEASE_DIR/${NOOBS_DISTRO}/${NOOBS_DISTRO}.png + fi + cp -PR $ROOT/config/noobs/os.json $RELEASE_DIR/${NOOBS_DISTRO} + cp -PR $ROOT/config/noobs/partition_setup.sh $RELEASE_DIR/${NOOBS_DISTRO} + cp -PR $ROOT/config/noobs/partitions.json $RELEASE_DIR/${NOOBS_DISTRO} + if [ -d $DISTRO_DIR/$DISTRO/noobs/marketing ]; then + tar cf $RELEASE_DIR/${NOOBS_DISTRO}/marketing.tar -C $DISTRO_DIR/$DISTRO/noobs/marketing . + else + tar cf $RELEASE_DIR/${NOOBS_DISTRO}/marketing.tar -C $ROOT/config/noobs/marketing . + fi + cp $ROOT/README* $RELEASE_DIR/${NOOBS_DISTRO} + cp $ROOT/CHANGELOG $RELEASE_DIR/${NOOBS_DISTRO}/release_notes.txt + + sed -e "s%@DISTRONAME@%$DISTRONAME%g" \ + -e "s%@PROJECT@%${DEVICE:-$PROJECT}%g" \ + -e "s%@LIBREELEC_VERSION@%$LIBREELEC_VERSION%g" \ + -e "s%@RELEASE_DATE@%$(date +%F)%g" \ + -e "s%@KERNEL_VERSION@%$(kernel_version)%g" \ + -e "s%@DESCRIPTION@%$DESCRIPTION%g" \ + -e "s%@ROOT_PASSWORD@%$ROOT_PASSWORD%g" \ + -e "s%@NOOBS_SUPPORTED_MODELS@%$NOOBS_SUPPORTED_MODELS%g" \ + -e "s%@NOOBS_HEX@%$NOOBS_HEX%g" \ + -i $RELEASE_DIR/${NOOBS_DISTRO}/os.json + + sed -e "s%@DISTRONAME@%$DISTRONAME%g" \ + -e "s%@PROJECT@%${DEVICE:-$PROJECT}%g" \ + -e "s%@SYSTEM_SIZE@%$SYSTEM_SIZE%g" \ + -i $RELEASE_DIR/${NOOBS_DISTRO}/partitions.json + + # Create System dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System + + BOOTLOADER_DIR="$(get_pkg_directory "$BOOTLOADER")" + if [ -d $BOOTLOADER_DIR/files/3rdparty/bootloader/ ]; then + cp -PR $BOOTLOADER_DIR/files/3rdparty/bootloader/* $RELEASE_DIR/${NOOBS_DISTRO}/System + fi + + # Copy Bootloader + cp -PR $BUILD/bcm2835-bootloader-*/LICENCE* $RELEASE_DIR/${NOOBS_DISTRO}/System/ + cp -PR $BUILD/bcm2835-bootloader-*/bootcode.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/ + cp -PR $BUILD/bcm2835-bootloader-*/fixup_x.dat $RELEASE_DIR/${NOOBS_DISTRO}/System/fixup.dat + cp -PR $BUILD/bcm2835-bootloader-*/start_x.elf $RELEASE_DIR/${NOOBS_DISTRO}/System/start.elf + [ -f $BUILD/bcm2835-bootloader-*/dt-blob.bin ] && cp -PR $BUILD/bcm2835-bootloader-*/dt-blob.bin $RELEASE_DIR/${NOOBS_DISTRO}/System/dt-blob.bin + + # Copy system files + cp $TARGET_IMG/$IMAGE_NAME.system $RELEASE_DIR/${NOOBS_DISTRO}/System/SYSTEM + cp $TARGET_IMG/$IMAGE_NAME.kernel $RELEASE_DIR/${NOOBS_DISTRO}/System/kernel.img + + for dtb in $INSTALL/usr/share/bootloader/*.dtb; do + if [ -f $dtb ]; then + cp -PR $dtb $RELEASE_DIR/${NOOBS_DISTRO}/System + fi + done + + for overlay in $INSTALL/usr/share/bootloader/overlays/*; do + if [ -f $overlay ]; then + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System/overlays + cp -PR $overlay $RELEASE_DIR/${NOOBS_DISTRO}/System/overlays + fi + done + + # Create md5sum's + ( cd $RELEASE_DIR/${NOOBS_DISTRO}/System; + md5sum -t SYSTEM > SYSTEM.md5; + md5sum -t kernel.img > kernel.img.md5; + ) + + # Copy additional files + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses + cp $ROOT/licenses/* $RELEASE_DIR/${NOOBS_DISTRO}/System/licenses + + # Create Storage dir + mkdir -p $RELEASE_DIR/${NOOBS_DISTRO}/Storage + + # Remove any previously created release tarball + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz + + # Create filesystem tarballs + tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/System.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/System/ . + tar cJf $RELEASE_DIR/${NOOBS_DISTRO}/Storage.tar.xz -C $RELEASE_DIR/${NOOBS_DISTRO}/Storage/ . + + # Remove filesystem dirs + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/System + rm -rf $RELEASE_DIR/${NOOBS_DISTRO}/Storage + + # Remove any previously created release tarball + rm -rf $TARGET_IMG/${IMAGE_NAME}-$1.tar + + # Create release tarball + tar cf $TARGET_IMG/${IMAGE_NAME}-$1.tar -C $TARGET_IMG ${IMAGE_NAME}-$1 + + # Create sha256 checksum of tarball + ( cd $TARGET_IMG + sha256sum ${IMAGE_NAME}-$1.tar > ${IMAGE_NAME}-$1.tar.sha256 + ) + fi + + if [ -d $RELEASE_DIR ]; then + # Cleanup release dir + rm -rf $RELEASE_DIR + fi +fi diff --git a/scripts/install b/scripts/install index cb8521ce49..375bb4c468 100755 --- a/scripts/install +++ b/scripts/install @@ -8,7 +8,7 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name" + die "usage: $0 package_name [parent_pkg]" fi if [ -z "${PKG_NAME}" ]; then @@ -29,29 +29,35 @@ if [ "${1//:/}" != "${1}" ]; then PACKAGE_NAME="${1%:*}" TARGET="${1#*:}" else - PACKAGE_NAME=$1 + PACKAGE_NAME="${1}" TARGET= fi [ -z "$TARGET" ] && TARGET="target" +PARENT_PKG="${2:-${PKG_NAME}:${TARGET}}" + +pkg_lock "${PACKAGE_NAME}:${TARGET}" "install" "${PARENT_PKG}" STAMP=$STAMPS_INSTALL/$PACKAGE_NAME/install_$TARGET +[ -f $STAMP ] && pkg_lock_status "UNLOCK" "${PACKAGE_NAME}:${TARGET}" "install" "already installed" [ -f $STAMP ] && exit 0 mkdir -p $STAMPS_INSTALL/$PACKAGE_NAME -$SCRIPTS/build $@ +$SCRIPTS/build "${1}" "${PARENT_PKG}" if [ "$TARGET" = target ] ; then for p in $PKG_DEPENDS_TARGET; do - $SCRIPTS/install $p + $SCRIPTS/install "${p}" "${PARENT_PKG}" done elif [ "$TARGET" = init ] ; then for p in $PKG_DEPENDS_INIT; do - $SCRIPTS/install $p + $SCRIPTS/install "${p}" "${PARENT_PKG}" done INSTALL=$BUILD/initramfs fi +pkg_lock_status "ACTIVE" "${PACKAGE_NAME}:${TARGET}" "install" + build_msg "CLR_INSTALL" "INSTALL" "${PACKAGE_NAME} $(print_color CLR_TARGET "(${TARGET})")" "indent" mkdir -p $INSTALL @@ -137,3 +143,5 @@ if [ "$TARGET" = target ] ; then fi touch $STAMP + +pkg_lock_status "UNLOCK" "${PACKAGE_NAME}:${TARGET}" "install" "installed" diff --git a/scripts/unpack b/scripts/unpack index 5da8a4ff77..84543366cf 100755 --- a/scripts/unpack +++ b/scripts/unpack @@ -7,14 +7,19 @@ . config/options "$1" if [ -z "$1" ]; then - die "usage: $0 package_name" + die "usage: $0 package_name [parent_pkg]" fi if [ -z "${PKG_NAME}" ]; then die "$(print_color CLR_ERROR "${1}: no package.mk file found")" fi +PARENT_PKG="${2:-${PKG_NAME}}" -$SCRIPTS/get $1 +pkg_lock "${PKG_NAME}" "unpack" "${PARENT_PKG}" + +pkg_lock_status "ACTIVE" "${PKG_NAME}" "unpack" + +$SCRIPTS/get "${PKG_NAME}" STAMP="$PKG_BUILD/.libreelec-unpack" @@ -22,13 +27,13 @@ mkdir -p $BUILD # Perform a wildcard match on the package to ensure old versions are cleaned too PKG_DEEPHASH= -for i in $BUILD/$1-*; do +for i in $BUILD/${PKG_NAME}-*; do if [ -d $i -a -f "$i/.libreelec-unpack" ] ; then . "$i/.libreelec-unpack" - if [ "$STAMP_PKG_NAME" = "$1" ]; then + if [ "$STAMP_PKG_NAME" = "${PKG_NAME}" ]; then [ -z "${PKG_DEEPHASH}" ] && PKG_DEEPHASH=$(calculate_stamp) if [ ! "$PKG_DEEPHASH" = "$STAMP_PKG_DEEPHASH" ] ; then - $SCRIPTS/clean $1 + $SCRIPTS/clean "${PKG_NAME}" fi fi fi @@ -36,13 +41,16 @@ done if [ -d "$PKG_BUILD" -a ! -f "$STAMP" ]; then # stale pkg build dir - $SCRIPTS/clean $1 + $SCRIPTS/clean "${PKG_NAME}" fi -[ -f "$STAMP" ] && exit 0 +if [ -f "$STAMP" ]; then + pkg_lock_status "UNLOCK" "${PKG_NAME}" "unpack" "already unpacked" + exit 0 +fi -if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then - build_msg "CLR_UNPACK" "UNPACK" "${1}" "indent" +if [ -d "$SOURCES/${PKG_NAME}" -o -d "$PKG_DIR/sources" ]; then + build_msg "CLR_UNPACK" "UNPACK" "${PKG_NAME}" "indent" pkg_call_exists pre_unpack && pkg_call pre_unpack @@ -50,7 +58,7 @@ if [ -d "$SOURCES/$1" -o -d "$PKG_DIR/sources" ]; then pkg_call unpack else if [ -n "$PKG_URL" ]; then - $SCRIPTS/extract $1 $BUILD + $SCRIPTS/extract "${PKG_NAME}" $BUILD fi fi @@ -161,10 +169,12 @@ fi if [ "$PKG_SECTION" != "virtual" ]; then mkdir -p "$PKG_BUILD" - rm -f $STAMPS/$1/build_* + rm -f $STAMPS/${PKG_NAME}/build_* PKG_DEEPHASH=$(calculate_stamp) for i in PKG_NAME PKG_DEEPHASH; do echo "STAMP_$i=\"${!i}\"" >> $STAMP done fi + +pkg_lock_status "UNLOCK" "${PKG_NAME}" "unpack" "unpacked" diff --git a/tools/dashboard b/tools/dashboard new file mode 100755 index 0000000000..3e78450e97 --- /dev/null +++ b/tools/dashboard @@ -0,0 +1,19 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +. config/options "" + +while [ : ]; do + echo "Waiting for build to start..." + while [ : ]; do + pid="$(cat "${THREAD_CONTROL}/pid" 2>/dev/null || true)" + [ -n "${pid}" ] && ps -p ${pid} &>/dev/null && break + sleep 1.0 + done + + tail -Fn+0 --pid=${pid} ${THREAD_CONTROL}/status 2>/dev/null + + echo +done diff --git a/tools/viewplan b/tools/viewplan new file mode 100755 index 0000000000..3b58af267b --- /dev/null +++ b/tools/viewplan @@ -0,0 +1,29 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +unset _CACHE_PACKAGE_LOCAL _CACHE_PACKAGE_GLOBAL _DEBUG_DEPENDS_LIST _DEBUG_PACKAGE_LIST + +. config/options "" +. config/multithread + +# Fake the parallel command if GNU parallel is not available - slow, but works. +fake_parallel() { + while read -r line; do + json_worker "${line}" + done +} + +PARALLEL_BIN=${TOOLCHAIN}/bin/parallel + +[ -x ${PARALLEL_BIN} ] || PARALLEL_BIN=parallel +command -v ${PARALLEL_BIN} >/dev/null || PARALLEL_BIN=fake_parallel + +# pipefail: return value of a pipeline is the value of the last (rightmost) command to exit with a non-zero status +set -o pipefail + +cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \ + ${PARALLEL_BIN} --plain --no-notice --max-args 30 json_worker --halt now,fail=1 | \ + ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \ + die "FAILURE: Unable to generate plan"