build: auto remove build dirs

config/functions

@@ -94,6 +94,7 @@ print_color() {
       CLR_PATCH_DESC)   clr_actual="${boldwhite}";;
       CLR_TARGET)       clr_actual="${boldwhite}";;
       CLR_UNPACK)       clr_actual="${boldcyan}";;
+      CLR_AUTOREMOVE)   clr_actual="${boldblue}";;
 
       CLR_ENDCOLOR)     clr_actual="${endcolor}";;
 

config/multithread

@@ -25,6 +25,8 @@ start_multithread_build() {
   fi
   buildopts+=" --log-combine ${LOGCOMBINE:-always}"
 
+  [ "${AUTOREMOVE}" = "yes" ] && buildopts+=" --auto-remove"
+
   # When building addons, don't halt on error - keep building all packages/addons
   [ "${MTADDONBUILD}" = "yes" ] && buildopts+=" --continue-on-error" || buildopts+=" --halt-on-error"
 

scripts/autoremove

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: GPL-2.0
+# Copyright (C) 2019-present Team LibreELEC (https://libreelec.tv)
+
+. config/options "${1}"
+
+if [ -d "${PKG_BUILD}" -a "${PKG_SECTION}" != "virtual" ]; then
+  print_color CLR_AUTOREMOVE "AUTOREMOVE ${PKG_BUILD}"
+  echo
+  rm -r "${PKG_BUILD}"
+fi

scripts/genbuildplan.py

@@ -19,6 +19,8 @@ class LibreELEC_Package:
         self.wants = []
         self.wantedby = []
 
+        self.unpacks = []
+
     def __repr__(self):
         s = "%-9s: %s" % ("name", self.name)
         s = "%s\n%-9s: %s" % (s, "section", self.section)
@@ -26,6 +28,8 @@ class LibreELEC_Package:
         for t in self.deps:
             s = "%s\n%-9s: %s" % (s, t, self.deps[t])
 
+        s = "%s\n%-9s: %s" % (s, "UNPACKS", self.unpacks)
+
         s = "%s\n%-9s: %s" % (s, "NEEDS", self.wants)
         s = "%s\n%-9s: %s" % (s, "WANTED BY", self.wantedby)
 
@@ -55,6 +59,10 @@ class LibreELEC_Package:
         if name in self.wantedby:
             self.wantedby.remove(name)
 
+    def addUnpack(self, packages):
+        if packages.strip():
+            self.unpacks = packages.strip().split()
+
     def isReferenced(self):
         return False if self.wants == [] else True
 
@@ -106,7 +114,8 @@ class Node:
         return self.name if self.target == "target" else "%s:%s" % (self.name, self.target)
 
     def addEdge(self, node):
-        self.edges.append(node)
+        if node not in self.edges:
+            self.edges.append(node)
 
 def eprint(*args, **kwargs):
     print(*args, file=sys.stderr, **kwargs)
@@ -136,6 +145,8 @@ def initPackage(package):
     for target in ["bootstrap", "init", "host", "target"]:
         pkg.addDependencies(target, package[target])
 
+    pkg.addUnpack(package["unpack"])
+
     return pkg
 
 # Split name:target into components
@@ -236,7 +247,8 @@ def processPackages(args, packages):
             "bootstrap": "",
             "init": "",
             "host": " ".join(get_packages_by_target("host", args.build)),
-            "target": " ".join(get_packages_by_target("target", args.build))
+            "target": " ".join(get_packages_by_target("target", args.build)),
+            "unpack": ""
           }
 
     packages[pkg["name"]] = initPackage(pkg)
@@ -366,9 +378,12 @@ eprint("")
 if args.with_json:
     plan = []
     for step in steps:
+        (pkg_name, target) = split_package(step[1])
         plan.append({"task": step[0],
                      "name": step[1],
-                     "deps": [d.fqname for d in REQUIRED_PKGS[step[1]].edges]})
+                     "section": ALL_PACKAGES[pkg_name].section,
+                     "wants": [d.fqname for d in REQUIRED_PKGS[step[1]].edges],
+                     "unpacks": ALL_PACKAGES[pkg_name].unpacks if pkg_name in ALL_PACKAGES else []})
 
     with open(args.with_json, "w") as out:
         print(json.dumps(plan, indent=2, sort_keys=False), file=out)

scripts/pkgbuilder.py

@@ -63,18 +63,71 @@ class Generator:
         self.building = {}
         self.built = {}
         self.failed = {}
+        self.removedPackages = {}
 
         self.check_no_deps = True
 
+        # Transform unpack info from package:target to just package - simplifying refcount generation
+        # Create a map for sections, as we don't autoremove "virtual" packages
+        self.unpacks = {}
+        self.sections = {}
+        for job in self.work:
+            (pkg_name, target) = job["name"].split(":")
+            if pkg_name not in self.unpacks:
+                self.unpacks[pkg_name] = job["unpacks"]
+                self.sections[pkg_name] = job["section"]
+                for unpack in job["unpacks"]:
+                    if unpack not in self.sections:
+                        self.sections[unpack] = "" # don't know section, assume not virtual
+
+        # Count number of times each package is referenced by package:target (including itself) and
+        # then recursively accumulate counts for any other packages that may be referenced
+        # by "PKG_DEPENDS_UNPACK".
+        # Once the refcount is zero for a package, the source directory can be removed.
+        self.refcount = {}
+        for job in self.work:
+            (pkg_name, target) = job["name"].split(":")
+            self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) + 1
+            for pkg_name in job["unpacks"]:
+                self.addRefCounts(pkg_name)
+
     def canBuildJob(self, job):
-        for dep in job["deps"]:
+        for dep in job["wants"]:
             if dep not in self.built:
                 return False
 
         return True
 
+    def getPackagesToRemove(self, job):
+        packages = {}
+
+        pkg_name = job["name"].split(":")[0]
+        packages[pkg_name] = True
+        for pkg_name in job["unpacks"]:
+            self.addUnpackPackages(pkg_name, packages)
+
+        for pkg_name in packages:
+            if self.refcount[pkg_name] == 0 and \
+                self.sections[pkg_name] != "virtual" and \
+                pkg_name not in self.removedPackages:
+                yield pkg_name
+
+    def getPackageReferenceCounts(self, job):
+        packages = {}
+
+        pkg_name = job["name"].split(":")[0]
+        packages[pkg_name] = True
+        for pkg_name in job["unpacks"]:
+            self.addUnpackPackages(pkg_name, packages)
+
+        for pkg_name in packages:
+            tokens = ""
+            tokens += "[v]" if self.sections[pkg_name] == "virtual" else ""
+            tokens += "[r]" if pkg_name in self.removedPackages else ""
+            yield("%s:%d%s" % (pkg_name, self.refcount[pkg_name], tokens))
+
     def getFirstFailedJob(self, job):
-        for dep in job["deps"]:
+        for dep in job["wants"]:
             if dep in self.failed:
                 failedjob = self.getFirstFailedJob(self.failed[dep])
                 if not failedjob:
@@ -86,7 +139,7 @@ class Generator:
 
     def getAllFailedJobs(self, job):
         flist = {}
-        for dep in job["deps"]:
+        for dep in job["wants"]:
             if dep in self.failed:
                 failedjob = self.getFirstFailedJob(self.failed[dep])
                 if failedjob:
@@ -104,7 +157,7 @@ class Generator:
         # until we're sure there's none left...
         if self.check_no_deps:
             for i, job in enumerate(self.work):
-                if job["deps"] == []:
+                if job["wants"] == []:
                     self.building[job["name"]] = True
                     del self.work[i]
                     job["failedjobs"] = self.getAllFailedJobs(job)
@@ -133,11 +186,11 @@ class Generator:
     # currently building jobs are complete.
     def getStallInfo(self):
         for job in self.work:
-            for dep in job["deps"]:
+            for dep in job["wants"]:
                 if dep not in self.building and dep not in self.built:
                     break
             else:
-                yield (job["name"], [d for d in job["deps"] if d in self.building])
+                yield (job["name"], [d for d in job["wants"] if d in self.building])
 
     def activeJobCount(self):
         return len(self.building)
@@ -157,10 +210,37 @@ class Generator:
         return self.totalJobs
 
     def completed(self, job):
-        del self.building[job["name"]]
         self.built[job["name"]] = job
+        del self.building[job["name"]]
+
         if job["failed"]:
             self.failed[job["name"]] = job
+        else:
+            self.refcount[job["name"].split(":")[0]] -= 1
+
+        for pkg_name in job["unpacks"]:
+            self.delRefCounts(pkg_name)
+
+    def removed(self, pkg_name):
+        self.removedPackages[pkg_name] = True
+
+    def addUnpackPackages(self, pkg_name, packages):
+        packages[pkg_name] = True
+        if pkg_name in self.unpacks:
+            for p in self.unpacks[pkg_name]:
+                self.addUnpackPackages(p, packages)
+
+    def addRefCounts(self, pkg_name):
+        self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) + 1
+        if pkg_name in self.unpacks:
+            for p in self.unpacks[pkg_name]:
+                self.addRefCounts(p)
+
+    def delRefCounts(self, pkg_name):
+        self.refcount[pkg_name] = self.refcount.get(pkg_name, 0) - 1
+        if pkg_name in self.unpacks:
+            for p in self.unpacks[pkg_name]:
+                self.delRefCounts(p)
 
 class BuildProcess(threading.Thread):
     def __init__(self, slot, maxslot, jobtotal, haltonerror, work, complete):
@@ -259,8 +339,8 @@ class BuildProcess(threading.Thread):
 
 class Builder:
     def __init__(self, maxthreadcount, inputfilename, jobglog, loadstats, stats_interval, \
-                 haltonerror=True, failimmediately=True, log_burst=True, log_combine="always", bookends=True, \
+                 haltonerror=True, failimmediately=True, log_burst=True, log_combine="always", \
-                 debug=False, verbose=False, colors=False):
+                 autoremove=False, bookends=True, colors=False, debug=False, verbose=False):
         if inputfilename == "-":
             plan = json.load(sys.stdin)
         else:
@@ -282,6 +362,7 @@ class Builder:
         self.debug = debug
         self.verbose = verbose
         self.bookends = bookends
+        self.autoremove = autoremove
 
         self.colors = (colors == "always" or (colors == "auto" and sys.stderr.isatty()))
         self.color_code = {}
@@ -345,6 +426,7 @@ class Builder:
             job = self.getCompletedJob()
 
             self.writeJobLog(job)
+            self.autoRemovePackages(job)
             self.processJobOutput(job)
             self.displayJobStatus(job)
 
@@ -538,6 +620,13 @@ class Builder:
                     if self.debug:
                         log_size += len(line)
 
+                if "autoremove" in job:
+                    for line in job["autoremove"].stdout:
+                        print(line, end="")
+                        if self.debug:
+                            log_size += len(line)
+                    job["autoremove"] = None
+
                 if self.bookends:
                     print(">>> %s" % job["name"])
 
@@ -561,6 +650,29 @@ class Builder:
                           j=job, prec=4, width=self.twidth),
                   file=self.joblogfile, flush=True)
 
+    # Remove any source code directories that are no longer required.
+    # Output from the subprocess is either appended to the burst logfile
+    # or is captured for later output to stdout (after the correspnding logfile).
+    def autoRemovePackages(self, job):
+        if self.autoremove:
+            if self.debug:
+                DEBUG("Cleaning Pkg: %s (%s)" % (job["name"], ", ".join(self.generator.getPackageReferenceCounts(job))))
+
+            for pkg_name in self.generator.getPackagesToRemove(job):
+                DEBUG("Removing Pkg: %s" % pkg_name)
+                args = ["%s/%s/autoremove" % (ROOT, SCRIPTS), pkg_name]
+                if job["logfile"]:
+                    with open(job["logfile"], "a") as logfile:
+                        cmd = subprocess.run(args, cwd=ROOT,
+                                             stdin=subprocess.PIPE, stdout=logfile, stderr=subprocess.STDOUT,
+                                             universal_newlines=True, shell=False)
+                else:
+                    job["autoremove"] = subprocess.run(args, cwd=ROOT,
+                                             stdin=subprocess.PIPE, capture_output=True,
+                                             universal_newlines=True, shell=False)
+
+                self.generator.removed(pkg_name)
+
     def startProcesses(self):
         for process in self.processes:
             process.start()
@@ -634,6 +746,9 @@ group.add_argument("--fail-immediately", action="store_true", default=True, \
 group.add_argument("--fail-after-active", action="store_false", dest="fail_immediately", \
                     help="With --halt-on-error, when an error occurs fail after all other active jobs have finished.")
 
+parser.add_argument("--auto-remove", action="store_true", default=False, \
+                    help="Automatically remove redundant source code directories. Default is disabled.")
+
 parser.add_argument("--verbose", action="store_true", default=False, \
                     help="Output verbose information to stderr.")
 
@@ -665,7 +780,8 @@ try:
     result = Builder(args.max_procs, args.plan, args.joblog, args.loadstats, args.stats_interval, \
                      haltonerror=args.halt_on_error, failimmediately=args.fail_immediately, \
                      log_burst=args.log_burst, log_combine=args.log_combine, bookends=args.with_bookends, \
-                     colors=args.colors, debug=args.debug, verbose=args.verbose).build()
+                     autoremove=args.auto_remove, colors=args.colors, \
+                     debug=args.debug, verbose=args.verbose).build()
 
     if DEBUG_LOG:
         DEBUG_LOG.close()

scripts/pkgjson

@@ -34,7 +34,8 @@ json_worker() {
     "bootstrap": "${PKG_DEPENDS_BOOTSTRAP}",
     "init": "${PKG_DEPENDS_INIT}",
     "host": "${PKG_DEPENDS_HOST}",
-    "target": "${PKG_DEPENDS_TARGET}"
+    "target": "${PKG_DEPENDS_TARGET}",
+    "unpack": "${PKG_DEPENDS_UNPACK}"
   },
 EOF
   done