mirror of
https://github.com/LibreELEC/LibreELEC.tv.git
synced 2025-07-24 11:16:51 +00:00
Merge pull request #4155 from MilhouseVH/le10_pkgbuilder_fail_slow
scripts/pkgbuilder.py: allow active jobs to finish once an error occurs
This commit is contained in:
Christian Hewitt
GitHub
commit
74d2b27891
@ -36,6 +36,7 @@ start_multithread_build() {
|
||||
buildopts+=" --colors=${MTCOLORS:-auto}"
|
||||
fi
|
||||
[ "${MTBOOKENDS}" = "no" ] && buildopts+=" --without-bookends" || buildopts+=" --with-bookends"
|
||||
[ "${MTIMMEDIATE}" = "no" ] && buildopts+=" --fail-after-active" || buildopts+=" --fail-immediately"
|
||||
|
||||
buildopts+=" --stats-interval ${MTINTERVAL:-60}"
|
||||
|
||||
|
||||
@ -10,7 +10,6 @@ import datetime, time
|
||||
import argparse
|
||||
import json
|
||||
import codecs
|
||||
import copy
|
||||
import threading
|
||||
import queue
|
||||
import subprocess
|
||||
@ -58,9 +57,9 @@ class GeneratorStalled(Exception):
|
||||
|
||||
class Generator:
|
||||
def __init__(self, plan):
|
||||
self.plan = plan
|
||||
self.work = plan
|
||||
|
||||
self.work = copy.deepcopy(self.plan)
|
||||
self.totalJobs = len(plan)
|
||||
self.building = {}
|
||||
self.built = {}
|
||||
self.failed = {}
|
||||
@ -150,8 +149,12 @@ class Generator:
|
||||
def failedJobCount(self):
|
||||
return len(self.failed)
|
||||
|
||||
def failedJobs(self):
|
||||
for name in self.failed:
|
||||
yield self.failed[name]
|
||||
|
||||
def totalJobCount(self):
|
||||
return len(self.plan)
|
||||
return self.totalJobs
|
||||
|
||||
def completed(self, job):
|
||||
del self.building[job["name"]]
|
||||
@ -224,7 +227,7 @@ class BuildProcess(threading.Thread):
|
||||
stdin=subprocess.PIPE, stdout=logfile, stderr=subprocess.STDOUT,
|
||||
universal_newlines=True, shell=False)
|
||||
returncode = cmd.returncode
|
||||
job["cmdproc" ] = cmd
|
||||
job["cmdproc"] = cmd
|
||||
else:
|
||||
try:
|
||||
cmd = rusage_run(job["args"], cwd=ROOT,
|
||||
@ -232,7 +235,7 @@ class BuildProcess(threading.Thread):
|
||||
universal_newlines=True, shell=False,
|
||||
encoding="utf-8", errors="replace")
|
||||
returncode = cmd.returncode
|
||||
job["cmdproc" ] = cmd
|
||||
job["cmdproc"] = cmd
|
||||
except UnicodeDecodeError:
|
||||
print('\nPKGBUILDER ERROR: UnicodeDecodeError while reading cmd.stdout from "%s %s"\n' % (job["task"], job["name"]), file=sys.stderr, flush=True)
|
||||
except Exception as e:
|
||||
@ -256,7 +259,7 @@ class BuildProcess(threading.Thread):
|
||||
|
||||
class Builder:
|
||||
def __init__(self, maxthreadcount, inputfilename, jobglog, loadstats, stats_interval, \
|
||||
haltonerror=True, log_burst=True, log_combine="always", bookends=True, \
|
||||
haltonerror=True, failimmediately=True, log_burst=True, log_combine="always", bookends=True, \
|
||||
debug=False, verbose=False, colors=False):
|
||||
if inputfilename == "-":
|
||||
plan = json.load(sys.stdin)
|
||||
@ -273,6 +276,7 @@ class Builder:
|
||||
self.stats_interval = 60
|
||||
|
||||
self.haltonerror = haltonerror
|
||||
self.failimmediately = failimmediately
|
||||
self.log_burst = log_burst
|
||||
self.log_combine = log_combine
|
||||
self.debug = debug
|
||||
@ -286,6 +290,7 @@ class Builder:
|
||||
self.color_code["ACTV"] = "\033[0;33m" #yellow
|
||||
self.color_code["IDLE"] = "\033[0;35m" #magenta
|
||||
self.color_code["INIT"] = "\033[0;36m" #cyan
|
||||
self.color_code["WAIT"] = "\033[0;35m" #magenta
|
||||
|
||||
self.work = queue.Queue()
|
||||
self.complete = queue.Queue()
|
||||
@ -313,7 +318,8 @@ class Builder:
|
||||
self.threadcount = int(maxthreadcount)
|
||||
|
||||
self.threadcount = 1 if self.threadcount < 1 else self.threadcount
|
||||
self.threadcount = self.jobtotal if self.jobtotal <= self.threadcount else self.threadcount
|
||||
self.threadcount = min(self.jobtotal, self.threadcount)
|
||||
self.threadcount = max(1, self.threadcount)
|
||||
|
||||
if args.debug:
|
||||
DEBUG("THREADCOUNT#: input arg: %s, computed: %d" % (maxthreadcount, self.threadcount))
|
||||
@ -345,9 +351,6 @@ class Builder:
|
||||
job["cmdproc"] = None
|
||||
job = None
|
||||
|
||||
if self.generator.failedJobCount() != 0 and self.haltonerror:
|
||||
break
|
||||
|
||||
self.captureStats(finished=True)
|
||||
self.stopProcesses()
|
||||
|
||||
@ -357,12 +360,36 @@ class Builder:
|
||||
if self.loadstatsfile:
|
||||
self.loadstatsfile.close()
|
||||
|
||||
return (self.generator.failedJobCount() == 0)
|
||||
if self.generator.failedJobCount() != 0:
|
||||
if self.haltonerror and not self.failimmediately:
|
||||
failed = [job for job in self.generator.failedJobs() if job["logfile"]]
|
||||
if failed != []:
|
||||
print("\nThe following log(s) for this failure are available:", file=sys.stdout)
|
||||
for job in failed:
|
||||
print(" %s => %s" % (job["name"], job["logfile"]), file=sys.stdout)
|
||||
print("", file=sys.stdout)
|
||||
sys.stdout.flush()
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
# Fill work queue with enough jobs to keep all processes busy.
|
||||
# Return True while jobs remain available to build, or queued jobs are still building.
|
||||
# Return False once all jobs have been queued, and finished building.
|
||||
def queueWork(self):
|
||||
|
||||
# If an error has occurred and we are not ignoring errors, then return True
|
||||
# (but don't schedule new work) if we are to exit after all currently
|
||||
# active jobs have finished, otherwise return False.
|
||||
if self.haltonerror and self.generator.failedJobCount() != 0:
|
||||
if not self.failimmediately and self.generator.activeJobCount() != 0:
|
||||
freeslots = self.threadcount - self.generator.activeJobCount()
|
||||
self.vprint("WAIT", "waiting", ", ".join(self.generator.activeJobNames()))
|
||||
DEBUG("Waiting for : %d active, %d idle [%s]" % (self.generator.activeJobCount(), freeslots, ", ".join(self.generator.activeJobNames())))
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
try:
|
||||
for i in range(self.generator.activeJobCount(), self.threadcount):
|
||||
job = self.generator.getNextJob()
|
||||
@ -455,9 +482,7 @@ class Builder:
|
||||
# Output progress info, and links to any relevant logs
|
||||
def displayJobStatus(self, job):
|
||||
self.cseq += 1
|
||||
print("[%0*d/%0*d] [%s] %-7s %s" %
|
||||
(self.twidth, self.cseq, self.twidth, self.jobtotal,
|
||||
self.colorise(job["status"]), job["task"], job["name"]), file=sys.stderr, flush=True)
|
||||
self.vprint(job["status"], job["task"], job["name"], p1=self.cseq, p2=self.jobtotal)
|
||||
|
||||
if job["failed"]:
|
||||
if job["logfile"]:
|
||||
@ -602,6 +627,13 @@ group.add_argument("--halt-on-error", action="store_true", default=True, \
|
||||
group.add_argument("--continue-on-error", action="store_false", dest="halt_on_error", \
|
||||
help="Disable --halt-on-error and continue building.")
|
||||
|
||||
group = parser.add_mutually_exclusive_group()
|
||||
group.add_argument("--fail-immediately", action="store_true", default=True, \
|
||||
help="With --halt-on-error, the build can either fail immediately or only after all " \
|
||||
"other active jobs have finished. Default is to fail immediately.")
|
||||
group.add_argument("--fail-after-active", action="store_false", dest="fail_immediately", \
|
||||
help="With --halt-on-error, when an error occurs fail after all other active jobs have finished.")
|
||||
|
||||
parser.add_argument("--verbose", action="store_true", default=False, \
|
||||
help="Output verbose information to stderr.")
|
||||
|
||||
@ -631,7 +663,7 @@ with open("%s/parallel.pid" % THREAD_CONTROL, "w") as pid:
|
||||
|
||||
try:
|
||||
result = Builder(args.max_procs, args.plan, args.joblog, args.loadstats, args.stats_interval, \
|
||||
haltonerror=args.halt_on_error, \
|
||||
haltonerror=args.halt_on_error, failimmediately=args.fail_immediately, \
|
||||
log_burst=args.log_burst, log_combine=args.log_combine, bookends=args.with_bookends, \
|
||||
colors=args.colors, debug=args.debug, verbose=args.verbose).build()
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user