scripts/genbuildplan.py: drop --reorder; cleanup args handling, and other processing.

config/multithread

@@ -91,7 +91,7 @@ start_multithread_build() {
 
   cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
     ${TOOLCHAIN}/bin/parallel --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \
-    ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --with-json "${THREAD_CONTROL}"/plan.json \
+    ${SCRIPTS}/genbuildplan.py --show-wants --with-json "${THREAD_CONTROL}"/plan.json \
       --build ${@} > "${THREAD_CONTROL}"/plan || result=1
 
   if [ ${result} -eq 0 ]; then

scripts/genbuildplan.py

@@ -109,7 +109,7 @@ class Node:
         self.edges.append(node)
 
 def eprint(*args, **kwargs):
-        print(*args, file=sys.stderr, **kwargs)
+    print(*args, file=sys.stderr, **kwargs)
 
 # Read a JSON list of all possible packages from stdin
 def loadPackages():
@@ -178,7 +178,7 @@ def findbuildpos(node, list):
     return list.index(candidate) + 1 if candidate else -1
 
 # Resolve dependencies for a node
-def dep_resolve(node, resolved, unresolved, noreorder):
+def dep_resolve(node, resolved, unresolved):
     unresolved.append(node)
 
     for edge in node.edges:
@@ -186,19 +186,15 @@ def dep_resolve(node, resolved, unresolved, noreorder):
             if edge in unresolved:
                 raise Exception('Circular reference detected: %s -> %s\nRemove %s from %s package.mk::PKG_DEPENDS_%s' % \
                                 (node.fqname, edge.commonName(), edge.commonName(), node.name, node.target.upper()))
-            dep_resolve(edge, resolved, unresolved, noreorder)
+            dep_resolve(edge, resolved, unresolved)
 
     if node not in resolved:
-        pos = -1 if noreorder else findbuildpos(node, resolved)
+        resolved.append(node)
-        if pos != -1:
-            resolved.insert(pos, node)
-        else:
-            resolved.append(node)
 
     unresolved.remove(node)
 
 # Return a list of build steps for the trigger packages
-def get_build_steps(args, nodes, trigger_pkgs, built_pkgs):
+def get_build_steps(args, nodes):
     resolved = []
     unresolved = []
 
@@ -210,12 +206,12 @@ def get_build_steps(args, nodes, trigger_pkgs, built_pkgs):
     #
     install = True if "image" in args.build else False
 
-    for pkgname in [x for x in trigger_pkgs if x]:
+    for pkgname in [x for x in args.build if x]:
         if pkgname.find(":") == -1:
             pkgname = "%s:target" % pkgname
 
         if pkgname in nodes:
-            dep_resolve(nodes[pkgname], resolved, unresolved, args.no_reorder)
+            dep_resolve(nodes[pkgname], resolved, unresolved)
 
     # Abort if any references remain unresolved
     if unresolved != []:
@@ -226,14 +222,12 @@ def get_build_steps(args, nodes, trigger_pkgs, built_pkgs):
 
     # Output list of resolved dependencies
     for pkg in resolved:
-        if pkg.fqname not in built_pkgs:
+        task = "build" if pkg.fqname.endswith(":host") or pkg.fqname.endswith(":init") or not install else "install"
-            built_pkgs.append(pkg.fqname)
+        yield(task, pkg.fqname)
-            task = "build" if pkg.fqname.endswith(":host") or pkg.fqname.endswith(":init") or not install else "install"
-            yield(task, pkg.fqname)
 
 # Reduce the complete list of packages to a map of those packages that will
 # be needed for the build.
-def processPackages(args, packages, build):
+def processPackages(args, packages):
     # Add dummy package to ensure build/install dependencies are not culled
     pkg = {
             "name": ROOT_PKG,
@@ -241,8 +235,8 @@ def processPackages(args, packages, build):
             "hierarchy": "global",
             "bootstrap": "",
             "init": "",
-            "host": " ".join(get_packages_by_target("host", build)),
+            "host": " ".join(get_packages_by_target("host", args.build)),
-            "target": " ".join(get_packages_by_target("target", build))
+            "target": " ".join(get_packages_by_target("target", args.build))
           }
 
     packages[pkg["name"]] = initPackage(pkg)
@@ -277,16 +271,17 @@ def processPackages(args, packages, build):
             needed_map[pkgname] = pkg
 
     # Validate package dependency references
-    for pkgname in needed_map:
+    if not args.ignore_invalid:
-        pkg = needed_map[pkgname]
+        for pkgname in needed_map:
-        for t in pkg.deps:
+            pkg = needed_map[pkgname]
-            for d in pkg.deps[t]:
+            for t in pkg.deps:
-                if split_package(d)[0] not in needed_map and not args.ignore_invalid:
+                for d in pkg.deps[t]:
-                    msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper())
+                    if split_package(d)[0] not in needed_map:
-                    if args.warn_invalid:
+                        msg = 'Invalid package reference: dependency %s in package %s::PKG_DEPENDS_%s is not valid' % (d, pkgname, t.upper())
-                        eprint("WARNING: %s" % msg)
+                        if args.warn_invalid:
-                    else:
+                            eprint("WARNING: %s" % msg)
-                        raise Exception(msg)
+                        else:
+                            raise Exception(msg)
 
     node_map = {}
 
@@ -336,24 +331,18 @@ parser = argparse.ArgumentParser(description="Generate package dependency list f
 parser.add_argument("-b", "--build", nargs="+", metavar="PACKAGE", required=True, \
                     help="Space-separated list of build trigger packages, either for host or target. Required property - specify at least one package.")
 
-parser.add_argument("--warn-invalid", action="store_true", \
+parser.add_argument("--warn-invalid", action="store_true", default=False, \
                     help="Warn about invalid/missing dependency packages, perhaps excluded by a PKG_ARCH incompatability. Default is to abort.")
 
-parser.add_argument("--no-reorder", action="store_true", default="True", \
+parser.add_argument("--ignore-invalid", action="store_true", default=False, \
-                    help="Do not resequence steps based on dependencies. This is the default.")
-
-parser.add_argument("--reorder", action="store_false", dest="no_reorder", \
-                    help="Disable --no-reorder and resequence packages to try and reduce stalls etc.")
-
-parser.add_argument("--show-wants", action="store_true", \
-                    help="Output \"wants\" dependencies for each step.")
-
-parser.add_argument("--hide-wants", action="store_false", dest="show_wants", default="True", \
-                    help="Disable --show-wants.")
-
-parser.add_argument("--ignore-invalid", action="store_true", \
                     help="Ignore invalid packages.")
 
+group =  parser.add_mutually_exclusive_group()
+group.add_argument("--show-wants", action="store_true", \
+                    help="Output \"wants\" dependencies for each step.")
+group.add_argument("--hide-wants", action="store_false", dest="show_wants", default=True, \
+                    help="Disable --show-wants.  This is the default.")
+
 parser.add_argument("--with-json", metavar="FILE", \
                     help="File into which JSON formatted plan will be written.")
 
@@ -363,14 +352,10 @@ ALL_PACKAGES = loadPackages()
 
 loaded = len(ALL_PACKAGES)
 
-REQUIRED_PKGS = processPackages(args, ALL_PACKAGES, args.build)
+REQUIRED_PKGS = processPackages(args, ALL_PACKAGES)
 
-# Output list of packages to build/install
+# Identify list of packages to build/install
-built_pkgs = []
+steps = [step for step in get_build_steps(args, REQUIRED_PKGS)]
-steps = []
-
-for step in get_build_steps(args, REQUIRED_PKGS, args.build, built_pkgs):
-    steps.append(step)
 
 eprint("Packages loaded : %d" % loaded)
 eprint("Build trigger(s): %d [%s]" % (len(args.build), " ".join(args.build)))
@@ -391,10 +376,8 @@ if args.with_json:
 # Output build/install steps
 if args.show_wants:
     for step in steps:
-        wants = []
         node = (REQUIRED_PKGS[step[1]])
-        for e in node.edges:
+        wants = [edge.fqname for edge in node.edges]
-            wants.append(e.fqname)
         print("%-7s %-25s (wants: %s)" % (step[0], step[1].replace(":target",""), ", ".join(wants).replace(":target","")))
 else:
     for step in steps:

tools/viewplan

@@ -25,5 +25,5 @@ set -o pipefail
 
 cat ${_CACHE_PACKAGE_GLOBAL} ${_CACHE_PACKAGE_LOCAL} | \
   ${PARALLEL_BIN} --plain --no-notice --max-args 30 --halt now,fail=1 json_worker | \
-  ${SCRIPTS}/genbuildplan.py --no-reorder --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \
+  ${SCRIPTS}/genbuildplan.py --show-wants --build ${@:-image} --warn-invalid ${GENFLAGS} || \
     die "FAILURE: Unable to generate plan"