From a11e063083ead554c87699bcaea9a7a6d212171c Mon Sep 17 00:00:00 2001 From: MilhouseVH Date: Thu, 30 Jan 2020 04:49:29 +0000 Subject: [PATCH] build: auto remove build dirs --- config/functions | 1 + config/multithread | 2 + scripts/autoremove | 12 ++++ scripts/genbuildplan.py | 21 ++++++- scripts/pkgbuilder.py | 136 +++++++++++++++++++++++++++++++++++++--- scripts/pkgjson | 3 +- 6 files changed, 161 insertions(+), 14 deletions(-) create mode 100755 scripts/autoremove diff --git a/config/functions b/config/functions index 16e78d7bcc..e363649d39 100644 --- a/config/functions +++ b/config/functions @@ -94,6 +94,7 @@ print_color() { CLR_PATCH_DESC) clr_actual="${boldwhite}";; CLR_TARGET) clr_actual="${boldwhite}";; CLR_UNPACK) clr_actual="${boldcyan}";; + CLR_AUTOREMOVE) clr_actual="${boldblue}";; CLR_ENDCOLOR) clr_actual="${endcolor}";; diff --git a/config/multithread b/config/multithread index 8b55858da4..0440964876 100644 --- a/config/multithread +++ b/config/multithread @@ -25,6 +25,8 @@ start_multithread_build() { fi buildopts+=" --log-combine ${LOGCOMBINE:-always}" + [ "${AUTOREMOVE}" = "yes" ] && buildopts+=" --auto-remove" + # When building addons, don't halt on error - keep building all packages/addons [ "${MTADDONBUILD}" = "yes" ] && buildopts+=" --continue-on-error" || buildopts+=" --halt-on-error" diff --git a/scripts/autoremove b/scripts/autoremove new file mode 100755 index 0000000000..ac57465489 --- /dev/null +++ b/scripts/autoremove @@ -0,0 +1,12 @@ +#!/bin/bash + +# SPDX-License-Identifier: GPL-2.0 +# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv) + +. config/options "${1}" + +if [ -d "${PKG_BUILD}" -a "${PKG_SECTION}" != "virtual" ]; then + print_color CLR_AUTOREMOVE "AUTOREMOVE ${PKG_BUILD}" + echo + rm -r "${PKG_BUILD}" +fi diff --git a/scripts/genbuildplan.py b/scripts/genbuildplan.py index 84adf8925d..82918b6076 100755 --- a/scripts/genbuildplan.py +++ b/scripts/genbuildplan.py @@ -19,6 +19,8 @@ class LibreELEC_Package: self.wants = [] self.wantedby = [] + self.unpacks = [] + def __repr__(self): s = "%-9s: %s" % ("name", self.name) s = "%s\n%-9s: %s" % (s, "section", self.section) @@ -26,6 +28,8 @@ class LibreELEC_Package: for t in self.deps: s = "%s\n%-9s: %s" % (s, t, self.deps[t]) + s = "%s\n%-9s: %s" % (s, "UNPACKS", self.unpacks) + s = "%s\n%-9s: %s" % (s, "NEEDS", self.wants) s = "%s\n%-9s: %s" % (s, "WANTED BY", self.wantedby) @@ -55,6 +59,10 @@ class LibreELEC_Package: if name in self.wantedby: self.wantedby.remove(name) + def addUnpack(self, packages): + if packages.strip(): + self.unpacks = packages.strip().split() + def isReferenced(self): return False if self.wants == [] else True @@ -106,7 +114,8 @@ class Node: return self.name if self.target == "target" else "%s:%s" % (self.name, self.target) def addEdge(self, node): - self.edges.append(node) + if node not in self.edges: + self.edges.append(node) def eprint(*args, **kwargs): print(*args, file=sys.stderr, **kwargs) @@ -136,6 +145,8 @@ def initPackage(package): for target in ["bootstrap", "init", "host", "target"]: pkg.addDependencies(target, package[target]) + pkg.addUnpack(package["unpack"]) + return pkg # Split name:target into components @@ -236,7 +247,8 @@ def processPackages(args, packages): "bootstrap": "", "init": "", "host": " ".join(get_packages_by_target("host", args.build)), - "target": " ".join(get_packages_by_target("target", args.build)) + "target": " ".join(get_packages_by_target("target", args.build)), + "unpack": "" } packages[pkg["name"]] = initPackage(pkg) @@ -366,9 +378,12 @@ eprint("") if args.with_json: plan = [] for step in steps: + (pkg_name, target) = split_package(step[1]) plan.append({"task": step[0], "name": step[1], - "deps": [d.fqname for d in REQUIRED_PKGS[step[1]].edges]}) + "section": ALL_PACKAGES[pkg_name].section, + "wants": [d.fqname for d in REQUIRED_PKGS[step[1]].edges], + "unpacks": ALL_PACKAGES[pkg_name].unpacks if pkg_name in ALL_PACKAGES else []}) with open(args.with_json, "w") as out: print(json.dumps(plan, indent=2, sort_keys=False), file=out) diff --git a/scripts/pkgbuilder.py b/scripts/pkgbuilder.py index f4b2602576..be12d8a0ac 100755 --- a/scripts/pkgbuilder.py +++ b/scripts/pkgbuilder.py @@ -63,18 +63,71 @@ class Generator: self.building = {} self.built = {} self.failed = {} + self.removedPackages = {} self.check_no_deps = True + # Transform unpack info from package:target to just package - simplifying refcount generation + # Create a map for sections, as we don't autoremove "virtual" packages + self.unpacks = {} + self.sections = {} + for job in self.work: + (pkg_name, target) = job["name"].split(":") + if pkg_name not in self.unpacks: + self.unpacks[pkg_name] = job["unpacks"] + self.sections[pkg_name] = job["section"] + for unpack in job["unpacks"]: + if unpack not in self.sections: + self.sections[unpack] = "" # don't know section, assume not virtual + + # Count number of times each package is referenced by package:target (including itself) and + # then recursively accumulate counts for any other packages that may be referenced + # by "PKG_DEPENDS_UNPACK". + # Once the refcount is zero for a package, the source directory can be removed. + self.refcount = {} + for job in self.work: + (pkg_name, target) = job["name"].split(":") + self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) + 1 + for pkg_name in job["unpacks"]: + self.addRefCounts(pkg_name) + def canBuildJob(self, job): - for dep in job["deps"]: + for dep in job["wants"]: if dep not in self.built: return False return True + def getPackagesToRemove(self, job): + packages = {} + + pkg_name = job["name"].split(":")[0] + packages[pkg_name] = True + for pkg_name in job["unpacks"]: + self.addUnpackPackages(pkg_name, packages) + + for pkg_name in packages: + if self.refcount[pkg_name] == 0 and \ + self.sections[pkg_name] != "virtual" and \ + pkg_name not in self.removedPackages: + yield pkg_name + + def getPackageReferenceCounts(self, job): + packages = {} + + pkg_name = job["name"].split(":")[0] + packages[pkg_name] = True + for pkg_name in job["unpacks"]: + self.addUnpackPackages(pkg_name, packages) + + for pkg_name in packages: + tokens = "" + tokens += "[v]" if self.sections[pkg_name] == "virtual" else "" + tokens += "[r]" if pkg_name in self.removedPackages else "" + yield("%s:%d%s" % (pkg_name, self.refcount[pkg_name], tokens)) + def getFirstFailedJob(self, job): - for dep in job["deps"]: + for dep in job["wants"]: if dep in self.failed: failedjob = self.getFirstFailedJob(self.failed[dep]) if not failedjob: @@ -86,7 +139,7 @@ class Generator: def getAllFailedJobs(self, job): flist = {} - for dep in job["deps"]: + for dep in job["wants"]: if dep in self.failed: failedjob = self.getFirstFailedJob(self.failed[dep]) if failedjob: @@ -104,7 +157,7 @@ class Generator: # until we're sure there's none left... if self.check_no_deps: for i, job in enumerate(self.work): - if job["deps"] == []: + if job["wants"] == []: self.building[job["name"]] = True del self.work[i] job["failedjobs"] = self.getAllFailedJobs(job) @@ -133,11 +186,11 @@ class Generator: # currently building jobs are complete. def getStallInfo(self): for job in self.work: - for dep in job["deps"]: + for dep in job["wants"]: if dep not in self.building and dep not in self.built: break else: - yield (job["name"], [d for d in job["deps"] if d in self.building]) + yield (job["name"], [d for d in job["wants"] if d in self.building]) def activeJobCount(self): return len(self.building) @@ -157,10 +210,37 @@ class Generator: return self.totalJobs def completed(self, job): - del self.building[job["name"]] self.built[job["name"]] = job + del self.building[job["name"]] + if job["failed"]: self.failed[job["name"]] = job + else: + self.refcount[job["name"].split(":")[0]] -= 1 + + for pkg_name in job["unpacks"]: + self.delRefCounts(pkg_name) + + def removed(self, pkg_name): + self.removedPackages[pkg_name] = True + + def addUnpackPackages(self, pkg_name, packages): + packages[pkg_name] = True + if pkg_name in self.unpacks: + for p in self.unpacks[pkg_name]: + self.addUnpackPackages(p, packages) + + def addRefCounts(self, pkg_name): + self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) + 1 + if pkg_name in self.unpacks: + for p in self.unpacks[pkg_name]: + self.addRefCounts(p) + + def delRefCounts(self, pkg_name): + self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) - 1 + if pkg_name in self.unpacks: + for p in self.unpacks[pkg_name]: + self.delRefCounts(p) class BuildProcess(threading.Thread): def __init__(self, slot, maxslot, jobtotal, haltonerror, work, complete): @@ -259,8 +339,8 @@ class BuildProcess(threading.Thread): class Builder: def __init__(self, maxthreadcount, inputfilename, jobglog, loadstats, stats_interval, \ - haltonerror=True, failimmediately=True, log_burst=True, log_combine="always", bookends=True, \ - debug=False, verbose=False, colors=False): + haltonerror=True, failimmediately=True, log_burst=True, log_combine="always", \ + autoremove=False, bookends=True, colors=False, debug=False, verbose=False): if inputfilename == "-": plan = json.load(sys.stdin) else: @@ -282,6 +362,7 @@ class Builder: self.debug = debug self.verbose = verbose self.bookends = bookends + self.autoremove = autoremove self.colors = (colors == "always" or (colors == "auto" and sys.stderr.isatty())) self.color_code = {} @@ -345,6 +426,7 @@ class Builder: job = self.getCompletedJob() self.writeJobLog(job) + self.autoRemovePackages(job) self.processJobOutput(job) self.displayJobStatus(job) @@ -538,6 +620,13 @@ class Builder: if self.debug: log_size += len(line) + if "autoremove" in job: + for line in job["autoremove"].stdout: + print(line, end="") + if self.debug: + log_size += len(line) + job["autoremove"] = None + if self.bookends: print(">>> %s" % job["name"]) @@ -561,6 +650,29 @@ class Builder: j=job, prec=4, width=self.twidth), file=self.joblogfile, flush=True) + # Remove any source code directories that are no longer required. + # Output from the subprocess is either appended to the burst logfile + # or is captured for later output to stdout (after the correspnding logfile). + def autoRemovePackages(self, job): + if self.autoremove: + if self.debug: + DEBUG("Cleaning Pkg: %s (%s)" % (job["name"], ", ".join(self.generator.getPackageReferenceCounts(job)))) + + for pkg_name in self.generator.getPackagesToRemove(job): + DEBUG("Removing Pkg: %s" % pkg_name) + args = ["%s/%s/autoremove" % (ROOT, SCRIPTS), pkg_name] + if job["logfile"]: + with open(job["logfile"], "a") as logfile: + cmd = subprocess.run(args, cwd=ROOT, + stdin=subprocess.PIPE, stdout=logfile, stderr=subprocess.STDOUT, + universal_newlines=True, shell=False) + else: + job["autoremove"] = subprocess.run(args, cwd=ROOT, + stdin=subprocess.PIPE, capture_output=True, + universal_newlines=True, shell=False) + + self.generator.removed(pkg_name) + def startProcesses(self): for process in self.processes: process.start() @@ -634,6 +746,9 @@ group.add_argument("--fail-immediately", action="store_true", default=True, \ group.add_argument("--fail-after-active", action="store_false", dest="fail_immediately", \ help="With --halt-on-error, when an error occurs fail after all other active jobs have finished.") +parser.add_argument("--auto-remove", action="store_true", default=False, \ + help="Automatically remove redundant source code directories. Default is disabled.") + parser.add_argument("--verbose", action="store_true", default=False, \ help="Output verbose information to stderr.") @@ -665,7 +780,8 @@ try: result = Builder(args.max_procs, args.plan, args.joblog, args.loadstats, args.stats_interval, \ haltonerror=args.halt_on_error, failimmediately=args.fail_immediately, \ log_burst=args.log_burst, log_combine=args.log_combine, bookends=args.with_bookends, \ - colors=args.colors, debug=args.debug, verbose=args.verbose).build() + autoremove=args.auto_remove, colors=args.colors, \ + debug=args.debug, verbose=args.verbose).build() if DEBUG_LOG: DEBUG_LOG.close() diff --git a/scripts/pkgjson b/scripts/pkgjson index 789fd51715..91b55efaa1 100755 --- a/scripts/pkgjson +++ b/scripts/pkgjson @@ -34,7 +34,8 @@ json_worker() { "bootstrap": "${PKG_DEPENDS_BOOTSTRAP}", "init": "${PKG_DEPENDS_INIT}", "host": "${PKG_DEPENDS_HOST}", - "target": "${PKG_DEPENDS_TARGET}" + "target": "${PKG_DEPENDS_TARGET}", + "unpack": "${PKG_DEPENDS_UNPACK}" }, EOF done