From 0c79812e60bfc9e057da6f4eff7c4a31c26b3daa Mon Sep 17 00:00:00 2001 From: David Given Date: Sat, 21 Sep 2024 00:21:39 +0200 Subject: [PATCH 1/2] Update ab. --- build/_progress.py | 5 ++ build/ab.mk | 6 ++ build/ab.py | 41 ++++++---- build/java.py | 185 --------------------------------------------- 4 files changed, 38 insertions(+), 199 deletions(-) create mode 100644 build/_progress.py delete mode 100644 build/java.py diff --git a/build/_progress.py b/build/_progress.py new file mode 100644 index 00000000..c930f0b5 --- /dev/null +++ b/build/_progress.py @@ -0,0 +1,5 @@ +import sys + +(_, current, max) = sys.argv +percent = int(100 * float(current) / float(max)) +print(f"[{percent:>3}%]") diff --git a/build/ab.mk b/build/ab.mk index ad1f532c..177d0edd 100644 --- a/build/ab.mk +++ b/build/ab.mk @@ -28,6 +28,12 @@ ifeq ($(OS), Windows_NT) endif EXT ?= +ifeq ($(PROGRESSINFO),) +rulecount := $(shell test -f $(OBJ)/build.mk && $(MAKE) -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l) +ruleindex := 1 +PROGRESSINFO = "$(shell $(PYTHON) build/_progress.py $(ruleindex) $(rulecount))$(eval ruleindex := $(shell expr $(ruleindex) + 1))" +endif + include $(OBJ)/build.mk MAKEFLAGS += -r -j$(shell nproc) diff --git a/build/ab.py b/build/ab.py index d9b4eef0..d10e7da9 100644 --- a/build/ab.py +++ b/build/ab.py @@ -16,6 +16,7 @@ import inspect import string import sys +import hashlib verbose = False quiet = False @@ -387,9 +388,12 @@ def filenameof(x): return xs[0] -def emit(*args): - outputFp.write(" ".join(args)) - outputFp.write("\n") +def emit(*args, into=None): + s = " ".join(args) + "\n" + if into is not None: + into += [s] + else: + outputFp.write(s) def emit_rule(name, ins, outs, cmds=[], label=None): @@ -398,26 +402,35 @@ def emit_rule(name, ins, outs, cmds=[], label=None): nonobjs = [f for f in fouts if not f.startswith("$(OBJ)")] emit("") + + lines = [] if nonobjs: - emit("clean::") - emit("\t$(hide) rm -f", *nonobjs) + emit("clean::", into=lines) + emit("\t$(hide) rm -f", *nonobjs, into=lines) - emit(".PHONY:", name) + emit(".PHONY:", name, into=lines) if outs: - emit(name, ":", *fouts) - if cmds: - emit(*fouts, "&:", *fins) - else: - emit(*fouts, ":", *fins) + emit(name, ":", *fouts, into=lines) + emit(*fouts, "&:", *fins, "\x01", into=lines) if label: - emit("\t$(hide)", "$(ECHO)", label) + emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO) ", label, into=lines) for c in cmds: - emit("\t$(hide)", c) + emit("\t$(hide)", c, into=lines) else: assert len(cmds) == 0, "rules with no outputs cannot have commands" - emit(name, ":", *fins) + emit(name, ":", *fins, into=lines) + + cmd = "".join(lines) + hash = hashlib.sha1(bytes(cmd, "utf-8")).hexdigest() + outputFp.write(cmd.replace("\x01", f"$(OBJ)/.hashes/{hash}")) + + if outs: + emit(f"$(OBJ)/.hashes/{hash}:") + emit( + f"\t$(hide) mkdir -p $(OBJ)/.hashes && touch $(OBJ)/.hashes/{hash}" + ) emit("") diff --git a/build/java.py b/build/java.py deleted file mode 100644 index d8a6905c..00000000 --- a/build/java.py +++ /dev/null @@ -1,185 +0,0 @@ -from build.ab import ( - simplerule, - error, - Rule, - Targets, - TargetsMap, - filenamesof, - filenameof, - emit, -) -from build.utils import targetswithtraitsof, collectattrs, filenamesmatchingof -from build.zip import zip -from os.path import * - -emit( - """ -JAR ?= jar -JAVAC ?= javac -JFLAGS ?= -g -""" -) - - -def _batched(items, n): - return (items[pos : pos + n] for pos in range(0, len(items), n)) - - -@Rule -def jar(self, name, items: TargetsMap = {}): - zip(replaces=self, items=items, extension="jar", flags="-0", label="JAR") - - -@Rule -def srcjar(self, name, items: TargetsMap = {}): - zip( - replaces=self, - items=items, - extension="srcjar", - flags="-0", - label="SRCJAR", - ) - - -@Rule -def externaljar(self, name, paths): - for f in paths: - if isfile(f): - simplerule( - replaces=self, - ins=[], - outs=[], - commands=[], - label="EXTERNALJAR", - args={"jar": f, "caller_deps": [self]}, - ) - return - error(f"None of {paths} exist") - - -@Rule -def javalibrary( - self, - name, - srcitems: TargetsMap = {}, - deps: Targets = [], -): - alldeps = collectattrs(targets=deps, name="caller_deps", initial=deps) - externaldeps = targetswithtraitsof(alldeps, "externaljar") - externaljars = [t.args["jar"] for t in externaldeps] - internaldeps = targetswithtraitsof(alldeps, "javalibrary") - srcdeps = targetswithtraitsof(alldeps, "srcjar") - - classpath = filenamesof(internaldeps) + externaljars - srcfiles = filenamesmatchingof(srcitems.values(), "*.java") - - cs = ( - # Setup. - [ - "rm -rf {dir}/src {dir}/objs {dir}/files.txt {outs[0]}", - "mkdir -p {dir}/src {dir}/objs", - ] - # Decompress any srcjars into directories of their own. - + [ - " && ".join( - [ - "(mkdir {dir}/src/" + str(i), - "cd {dir}/src/" + str(i), - "$(JAR) xf $(abspath " + f + "))", - ] - ) - for i, f in enumerate(filenamesof(srcdeps)) - ] - ) - - if srcfiles or srcdeps: - # Construct the list of filenames (which can be too long to go on - # the command line). - cs += ( - [ - "echo " + (" ".join(batch)) + " >> {dir}/files.txt" - for batch in _batched(srcfiles, 100) - ] - + ["find {dir}/src -name '*.java' >> {dir}/files.txt"] - # Actually do the compilation. - + [ - " ".join( - [ - "$(JAVAC)", - "$(JFLAGS)", - "-d {dir}/objs", - (" -cp " + ":".join(classpath)) if classpath else "", - "@{dir}/files.txt", - ] - ) - ] - ) - - # jar up the result. - cs += [ - "$(JAR) --create --no-compress --file {outs[0]} -C {self.dir}/objs ." - ] - - simplerule( - replaces=self, - ins=list(srcitems.values()) + deps, - outs=[f"={self.localname}.jar"], - commands=cs, - label="JAVALIBRARY", - args={"caller_deps": externaldeps + internaldeps}, - ) - - -@Rule -def javaprogram( - self, - name, - srcitems: TargetsMap = {}, - deps: Targets = [], - mainclass=None, -): - alldeps = collectattrs(targets=deps, name="caller_deps", initial=deps) - externaldeps = targetswithtraitsof(alldeps, "externaljar") - externaljars = [t.args["jar"] for t in externaldeps] - internaldeps = targetswithtraitsof(alldeps, "javalibrary") - - assert mainclass, "a main class must be specified for javaprogram" - if srcitems: - j = javalibrary( - name=name + "_mainlib", - srcitems=srcitems, - deps=deps, - cwd=self.cwd, - ) - j.materialise() - internaldeps += [j] - alldeps += [j] - - simplerule( - replaces=self, - ins=alldeps, - outs=[f"={self.localname}.jar"], - commands=[ - "rm -rf {dir}/objs", - "mkdir -p {dir}/objs", - "echo 'Manifest-Version: 1.0' > {dir}/manifest.mf", - "echo 'Created-By: ab' >> {dir}/manifest.mf", - "echo 'Main-Class: " + mainclass + "' >> {dir}/manifest.mf", - ] - + ( - ( - ["printf 'Class-Path:' >> {dir}/manifest.mf"] - + [f"echo ' {j}' >> {{dir}}/manifest.mf" for j in externaljars] - ) - if externaljars - else [] - ) - + [ - "(cd {dir}/objs && $(JAR) xf $(abspath " + j + "))" - for j in filenamesof(internaldeps) - ] - + [ - "$(JAR) --create --file={outs[0]} --manifest={dir}/manifest.mf -C {dir}/objs ." - ], - label="JAVAPROGRAM", - ) From b01f7f771dce3cda0a6041c73984aaef25c12887 Mon Sep 17 00:00:00 2001 From: David Given Date: Mon, 30 Sep 2024 22:36:06 +0200 Subject: [PATCH 2/2] Update ab. --- build/ab.mk | 7 ++++--- build/ab.py | 14 +++++++++++--- build/c.py | 2 +- build/utils.py | 11 ++++++----- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/build/ab.mk b/build/ab.mk index 177d0edd..e7c3172b 100644 --- a/build/ab.mk +++ b/build/ab.mk @@ -11,6 +11,7 @@ CFLAGS ?= -g -Og LDFLAGS ?= -g PKG_CONFIG ?= pkg-config ECHO ?= echo +CP ?= cp TARGETS ?= +all ifdef VERBOSE @@ -29,7 +30,7 @@ endif EXT ?= ifeq ($(PROGRESSINFO),) -rulecount := $(shell test -f $(OBJ)/build.mk && $(MAKE) -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l) +rulecount := $(shell $(MAKE) --no-print-directory -q $(OBJ)/build.mk PROGRESSINFO=1 && $(MAKE) -n $(MAKECMDGOALS) PROGRESSINFO=XXXPROGRESSINFOXXX | grep XXXPROGRESSINFOXXX | wc -l) ruleindex := 1 PROGRESSINFO = "$(shell $(PYTHON) build/_progress.py $(ruleindex) $(rulecount))$(eval ruleindex := $(shell expr $(ruleindex) + 1))" endif @@ -53,8 +54,8 @@ clean:: export PYTHONHASHSEED = 1 build-files = $(shell find . -name 'build.py') $(wildcard build/*.py) $(wildcard config.py) -$(OBJ)/build.mk: Makefile $(build-files) +$(OBJ)/build.mk: Makefile $(build-files) build/ab.mk @echo "AB" @mkdir -p $(OBJ) - $(hide) $(PYTHON) -X pycache_prefix=$(OBJ) build/ab.py -o $@ build.py \ + $(hide) $(PYTHON) -X pycache_prefix=$(OBJ)/__pycache__ build/ab.py -o $@ build.py \ || rm -f $@ diff --git a/build/ab.py b/build/ab.py index d10e7da9..4983c9ae 100644 --- a/build/ab.py +++ b/build/ab.py @@ -154,6 +154,9 @@ def __init__(self, cwd, name): def __eq__(self, other): return self.name is other.name + def __lt__(self, other): + return self.name < other.name + def __hash__(self): return id(self) @@ -314,7 +317,12 @@ def targetof(value, cwd=None): # Load the new build file. path = join(path, "build.py") - loadbuildfile(path) + try: + loadbuildfile(path) + except ModuleNotFoundError: + error( + f"no such build file '{path}' while trying to resolve '{value}'" + ) assert ( value in targets ), f"build file at '{path}' doesn't contain '+{target}' when trying to resolve '{value}'" @@ -411,7 +419,7 @@ def emit_rule(name, ins, outs, cmds=[], label=None): emit(".PHONY:", name, into=lines) if outs: emit(name, ":", *fouts, into=lines) - emit(*fouts, "&:", *fins, "\x01", into=lines) + emit(*fouts, "&:" if len(fouts) > 1 else ":", *fins, "\x01", into=lines) if label: emit("\t$(hide)", "$(ECHO) $(PROGRESSINFO) ", label, into=lines) @@ -489,7 +497,7 @@ def export(self, name=None, items: TargetsMap = {}, deps: Targets = []): cwd=self.cwd, ins=[srcs[0]], outs=[destf], - commands=["cp %s %s" % (srcs[0], destf)], + commands=["$(CP) %s %s" % (srcs[0], destf)], label="CP", ) subrule.materialise() diff --git a/build/c.py b/build/c.py index 71836fe8..52b942b7 100644 --- a/build/c.py +++ b/build/c.py @@ -139,7 +139,7 @@ def cheaders( len(s) == 1 ), "the target of a header must return exactly one file" - cs += ["cp {ins[" + str(i) + "]} {outs[" + str(i) + "]}"] + cs += ["$(CP) {ins[" + str(i) + "]} {outs[" + str(i) + "]}"] outs += ["=" + dest] i = i + 1 diff --git a/build/utils.py b/build/utils.py index ac57304e..17040a8f 100644 --- a/build/utils.py +++ b/build/utils.py @@ -8,8 +8,8 @@ error, simplerule, ) -from os.path import relpath, splitext, join, basename -from glob import glob +from os.path import relpath, splitext, join, basename, isfile +from glob import iglob import fnmatch import itertools @@ -30,7 +30,7 @@ def collectattrs(*, targets, name, initial=[]): s = set(initial) for a in [t.args.get(name, []) for t in targets]: s.update(a) - return list(s) + return sorted(list(s)) def itemsof(pattern, root=None, cwd=None): @@ -43,9 +43,10 @@ def itemsof(pattern, root=None, cwd=None): root = join(cwd, root) result = {} - for f in glob(pattern, recursive=True): + for f in iglob(pattern, recursive=True): try: - result[relpath(f, root)] = f + if isfile(f): + result[relpath(f, root)] = f except ValueError: error(f"file '{f}' is not in root '{root}'") return result