diff --git a/build/ab.mk b/build/ab.mk index 024cefa9..ad1f532c 100644 --- a/build/ab.mk +++ b/build/ab.mk @@ -30,7 +30,7 @@ EXT ?= include $(OBJ)/build.mk -MAKEFLAGS += -r +MAKEFLAGS += -r -j$(shell nproc) .DELETE_ON_ERROR: .PHONY: update-ab diff --git a/build/ab.py b/build/ab.py index 7637aa2c..d9b4eef0 100644 --- a/build/ab.py +++ b/build/ab.py @@ -112,6 +112,12 @@ def wrapper(*, name=None, replaces=None, **kwargs): t.types = func.__annotations__ t.callback = func t.traits.add(func.__name__) + if "args" in kwargs: + t.args |= kwargs["args"] + del kwargs["args"] + if "traits" in kwargs: + t.traits |= kwargs["traits"] + del kwargs["traits"] t.binding = sig.bind(name=name, self=t, **kwargs) t.binding.apply_defaults() @@ -125,6 +131,12 @@ def wrapper(*, name=None, replaces=None, **kwargs): return wrapper +def _isiterable(xs): + return isinstance(xs, Iterable) and not isinstance( + xs, (str, bytes, bytearray) + ) + + class Target: def __init__(self, cwd, name): if verbose: @@ -145,7 +157,7 @@ def __hash__(self): return id(self) def __repr__(self): - return f"Target('{self.name}', {id(self)})" + return f"Target('{self.name}')" def templateexpand(selfi, s): class Formatter(string.Formatter): @@ -160,7 +172,7 @@ def format_field(self, value, format_spec): return "" if type(value) == str: return value - if isinstance(value, (set, tuple)): + if _isiterable(value): value = list(value) if type(value) != list: value = [value] @@ -185,7 +197,6 @@ def materialise(self, replacing=False): # Perform type conversion to the declared rule parameter types. try: - self.args = {} for k, v in self.binding.arguments.items(): if k != "kwargs": t = self.types.get(k, None) @@ -205,9 +216,23 @@ def materialise(self, replacing=False): # Actually call the callback. cwdStack.append(self.cwd) - self.callback( - **{k: v for k, v in self.args.items() if k not in {"dir"}} - ) + if "kwargs" in self.binding.arguments.keys(): + # If the caller wants kwargs, return all arguments except the standard ones. + cbargs = { + k: v for k, v in self.args.items() if k not in {"dir"} + } + else: + # Otherwise, just call the callback with the ones it asks for. + cbargs = {} + for k in self.binding.arguments.keys(): + if k != "kwargs": + try: + cbargs[k] = self.args[k] + except KeyError: + error( + f"invocation of {self} failed because {k} isn't an argument" + ) + self.callback(**cbargs) cwdStack.pop() except BaseException as e: print(f"Error materialising {self}: {self.callback}") @@ -228,73 +253,82 @@ def convert(value, target): return target.targetof(value) def targetof(self, value): - if isinstance(value, Path): - value = value.as_posix() - if isinstance(value, Target): - t = value - else: - if value[0] == "=": - value = join(self.dir, value[1:]) - - if value.startswith("."): - # Check for local rule. - if value.startswith(".+"): - value = normpath(join(self.cwd, value[1:])) - # Check for local path. - elif value.startswith("./"): - value = normpath(join(self.cwd, value)) - # Explicit directories are always raw files. - elif value.endswith("/"): - return self._filetarget(value) - # Anything starting with a variable expansion is always a raw file. - elif value.startswith("$"): - return self._filetarget(value) - - # If this is not a rule lookup... - if "+" not in value: - # ...and if the value is pointing at a directory without a trailing /, - # it's a shorthand rule lookup. - if isdir(value): - value = value + "+" + basename(value) - # Otherwise it's an absolute file. - else: - return self._filetarget(value) + if isinstance(value, str) and (value[0] == "="): + value = join(self.dir, value[1:]) - # At this point we have the fully qualified name of a rule. + return targetof(value, self.cwd) - (path, target) = value.rsplit("+", 1) - value = join(path, "+" + target) - if value not in targets: - # Load the new build file. - path = join(path, "build.py") - loadbuildfile(path) - assert ( - value in targets - ), f"build file at '{path}' doesn't contain '+{target}' when trying to resolve '{value}'" +def _filetarget(value, cwd): + if value in targets: + return targets[value] - t = targets[value] + t = Target(cwd, value) + t.outs = [value] + targets[value] = t + return t - t.materialise() - return t - def _filetarget(self, value): - if value in targets: - return targets[value] +def targetof(value, cwd=None): + if not cwd: + cwd = cwdStack[-1] + if isinstance(value, Path): + value = value.as_posix() + if isinstance(value, Target): + t = value + else: + assert ( + value[0] != "=" + ), "can only use = for targets associated with another target" + + if value.startswith("."): + # Check for local rule. + if value.startswith(".+"): + value = normpath(join(cwd, value[1:])) + # Check for local path. + elif value.startswith("./"): + value = normpath(join(cwd, value)) + # Explicit directories are always raw files. + elif value.endswith("/"): + return _filetarget(value, cwd) + # Anything starting with a variable expansion is always a raw file. + elif value.startswith("$"): + return _filetarget(value, cwd) + + # If this is not a rule lookup... + if "+" not in value: + # ...and if the value is pointing at a directory without a trailing /, + # it's a shorthand rule lookup. + if isdir(value): + value = value + "+" + basename(value) + # Otherwise it's an absolute file. + else: + return _filetarget(value, cwd) - t = Target(self.cwd, value) - t.outs = [value] - targets[value] = t - return t + # At this point we have the fully qualified name of a rule. + + (path, target) = value.rsplit("+", 1) + value = join(path, "+" + target) + if value not in targets: + # Load the new build file. + + path = join(path, "build.py") + loadbuildfile(path) + assert ( + value in targets + ), f"build file at '{path}' doesn't contain '+{target}' when trying to resolve '{value}'" + + t = targets[value] + + t.materialise() + return t class Targets: def convert(value, target): if not value: return [] - assert isinstance( - value, (list, tuple) - ), "cannot convert non-list to Targets" + assert _isiterable(value), "cannot convert non-list to Targets" return [target.targetof(x) for x in flatten(value)] @@ -318,7 +352,7 @@ def loadbuildfile(filename): def flatten(items): def generate(xs): for x in xs: - if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + if _isiterable(x): yield from generate(x) else: yield x @@ -327,15 +361,13 @@ def generate(xs): def targetnamesof(items): - if not isinstance(items, (list, tuple, set)): - error("argument of filenamesof is not a list/tuple/set") + assert _isiterable(items), "argument of filenamesof is not a collection" return [t.name for t in items] def filenamesof(items): - if not isinstance(items, (list, tuple, set)): - error("argument of filenamesof is not a list/tuple/set") + assert _isiterable(items), "argument of filenamesof is not a collection" def generate(xs): for x in xs: @@ -398,7 +430,6 @@ def simplerule( deps: Targets = [], commands=[], label="RULE", - **kwargs, ): self.ins = ins self.outs = outs diff --git a/build/c.py b/build/c.py index b1185eb9..71836fe8 100644 --- a/build/c.py +++ b/build/c.py @@ -48,7 +48,7 @@ def cfileimpl(self, name, srcs, deps, suffix, commands, label, kind, cflags): outs=[outleaf], label=label, commands=commands, - cflags=cflags, + args={"cflags": cflags}, ) @@ -150,7 +150,7 @@ def cheaders( commands=cs, deps=deps, label="CHEADERS", - caller_cflags=caller_cflags + ["-I" + self.dir], + args={"caller_cflags": caller_cflags + ["-I" + self.dir]}, ) @@ -204,13 +204,18 @@ def libraryimpl( outs=[f"={self.localname}.a"], label=label, commands=commands, - caller_cflags=(hr.args["caller_cflags"] if hr else []), - caller_ldflags=caller_ldflags, + args={ + "caller_cflags": collectattrs( + targets=deps + ([hr] if hr else []), name="caller_cflags" + ), + "caller_ldflags": collectattrs( + targets=deps, name="caller_ldflags", initial=caller_ldflags + ), + }, + traits={"cheaders"}, ) self.outs = self.outs + (hr.outs if hr else []) - self.traits.add("cheaders") - @Rule def clibrary( @@ -310,9 +315,11 @@ def programimpl( deps=deps, label=toolchain.label + label, commands=commands, - ldflags=collectattrs( - targets=deps, name="caller_ldflags", initial=ldflags - ), + args={ + "ldflags": collectattrs( + targets=deps, name="caller_ldflags", initial=ldflags + ) + }, ) diff --git a/build/java.py b/build/java.py index 20f67ede..d8a6905c 100644 --- a/build/java.py +++ b/build/java.py @@ -1,14 +1,15 @@ from build.ab import ( simplerule, + error, Rule, Targets, TargetsMap, filenamesof, - error, filenameof, emit, ) -from build.utils import targetswithtraitsof, collectattrs +from build.utils import targetswithtraitsof, collectattrs, filenamesmatchingof +from build.zip import zip from os.path import * emit( @@ -20,87 +21,112 @@ ) +def _batched(items, n): + return (items[pos : pos + n] for pos in range(0, len(items), n)) + + @Rule -def jar(self, name, srcs: Targets = [], srcroot=None): - if not srcroot: - srcroot = self.cwd - fs = filenamesof(srcs) - try: - fs = [relpath(f, srcroot) for f in fs] - except ValueError: - error(f"some source files in {fs} aren't in the srcroot, {srcroot}") +def jar(self, name, items: TargetsMap = {}): + zip(replaces=self, items=items, extension="jar", flags="-0", label="JAR") - simplerule( + +@Rule +def srcjar(self, name, items: TargetsMap = {}): + zip( replaces=self, - ins=srcs, - outs=["=source.jar"], - commands=["jar cf {outs[0]} -C " + srcroot + " " + (" ".join(fs))], - label="JAR", + items=items, + extension="srcjar", + flags="-0", + label="SRCJAR", ) @Rule -def externaljar(self, name, path): - simplerule( - replaces=self, - ins=[], - outs=[], - commands=[], - label="EXTERNALJAR", - jar=path, - ) +def externaljar(self, name, paths): + for f in paths: + if isfile(f): + simplerule( + replaces=self, + ins=[], + outs=[], + commands=[], + label="EXTERNALJAR", + args={"jar": f, "caller_deps": [self]}, + ) + return + error(f"None of {paths} exist") @Rule def javalibrary( self, name, - srcs: Targets = [], - srcroot=None, - extrasrcs: TargetsMap = {}, + srcitems: TargetsMap = {}, deps: Targets = [], ): - filemap = {k: filenameof(v) for k, v in extrasrcs.items()} - ins = [] - for f in filenamesof(srcs): - try: - ff = relpath(f, srcroot) - except ValueError: - error(f"source file {f} is not in the srcroot {srcroot}") - filemap[ff] = f - ins += [f] - - jardeps = filenamesof(targetswithtraitsof(deps, "javalibrary")) + [ - t.args["jar"] for t in targetswithtraitsof(deps, "externaljar") - ] + alldeps = collectattrs(targets=deps, name="caller_deps", initial=deps) + externaldeps = targetswithtraitsof(alldeps, "externaljar") + externaljars = [t.args["jar"] for t in externaldeps] + internaldeps = targetswithtraitsof(alldeps, "javalibrary") + srcdeps = targetswithtraitsof(alldeps, "srcjar") + + classpath = filenamesof(internaldeps) + externaljars + srcfiles = filenamesmatchingof(srcitems.values(), "*.java") - dirs = {dirname(s) for s in filemap.keys()} cs = ( + # Setup. [ - "rm -rf {dir}/srcs {dir}/objs {outs[0]}", - "mkdir -p " + (" ".join([f"{self.dir}/srcs/{k}" for k in dirs])), + "rm -rf {dir}/src {dir}/objs {dir}/files.txt {outs[0]}", + "mkdir -p {dir}/src {dir}/objs", ] - + [f"cp {v} {self.dir}/srcs/{k}" for k, v in filemap.items()] + # Decompress any srcjars into directories of their own. + [ - " ".join( + " && ".join( [ - "$(JAVAC)", - "$(JFLAGS)", - "-d {dir}/objs", - " -cp " + (":".join(jardeps)) if jardeps else "", + "(mkdir {dir}/src/" + str(i), + "cd {dir}/src/" + str(i), + "$(JAR) xf $(abspath " + f + "))", ] - + [f"{self.dir}/srcs/{k}" for k in filemap.keys()] - ), - "$(JAR) --create --no-compress --file {outs[0]} -C {self.dir}/objs .", + ) + for i, f in enumerate(filenamesof(srcdeps)) ] ) + if srcfiles or srcdeps: + # Construct the list of filenames (which can be too long to go on + # the command line). + cs += ( + [ + "echo " + (" ".join(batch)) + " >> {dir}/files.txt" + for batch in _batched(srcfiles, 100) + ] + + ["find {dir}/src -name '*.java' >> {dir}/files.txt"] + # Actually do the compilation. + + [ + " ".join( + [ + "$(JAVAC)", + "$(JFLAGS)", + "-d {dir}/objs", + (" -cp " + ":".join(classpath)) if classpath else "", + "@{dir}/files.txt", + ] + ) + ] + ) + + # jar up the result. + cs += [ + "$(JAR) --create --no-compress --file {outs[0]} -C {self.dir}/objs ." + ] + simplerule( replaces=self, - ins=ins + deps, - outs=[f"={name}.jar"], + ins=list(srcitems.values()) + deps, + outs=[f"={self.localname}.jar"], commands=cs, label="JAVALIBRARY", + args={"caller_deps": externaldeps + internaldeps}, ) @@ -108,37 +134,52 @@ def javalibrary( def javaprogram( self, name, - srcs: Targets = [], - srcroot=None, - extrasrcs: TargetsMap = {}, + srcitems: TargetsMap = {}, deps: Targets = [], mainclass=None, ): - jars = filenamesof(targetswithtraitsof(deps, "javalibrary")) + alldeps = collectattrs(targets=deps, name="caller_deps", initial=deps) + externaldeps = targetswithtraitsof(alldeps, "externaljar") + externaljars = [t.args["jar"] for t in externaldeps] + internaldeps = targetswithtraitsof(alldeps, "javalibrary") assert mainclass, "a main class must be specified for javaprogram" - if srcs or extrasrcs: + if srcitems: j = javalibrary( name=name + "_mainlib", - srcs=srcs, - srcroot=srcroot, - extrasrcs=extrasrcs, + srcitems=srcitems, deps=deps, cwd=self.cwd, ) j.materialise() - jars += [filenameof(j)] + internaldeps += [j] + alldeps += [j] simplerule( replaces=self, - ins=jars, + ins=alldeps, outs=[f"={self.localname}.jar"], - commands=["rm -rf {dir}/objs", "mkdir -p {dir}/objs"] - + ["(cd {dir}/objs && $(JAR) xf $(abspath " + j + "))" for j in jars] + commands=[ + "rm -rf {dir}/objs", + "mkdir -p {dir}/objs", + "echo 'Manifest-Version: 1.0' > {dir}/manifest.mf", + "echo 'Created-By: ab' >> {dir}/manifest.mf", + "echo 'Main-Class: " + mainclass + "' >> {dir}/manifest.mf", + ] + + ( + ( + ["printf 'Class-Path:' >> {dir}/manifest.mf"] + + [f"echo ' {j}' >> {{dir}}/manifest.mf" for j in externaljars] + ) + if externaljars + else [] + ) + + [ + "(cd {dir}/objs && $(JAR) xf $(abspath " + j + "))" + for j in filenamesof(internaldeps) + ] + [ - "$(JAR) --create --file={outs[0]} --main-class=" - + mainclass - + " -C {dir}/objs ." + "$(JAR) --create --file={outs[0]} --manifest={dir}/manifest.mf -C {dir}/objs ." ], - label="MERGEJARS", + label="JAVAPROGRAM", ) diff --git a/build/protobuf.py b/build/protobuf.py index 65dc8d53..def6807a 100644 --- a/build/protobuf.py +++ b/build/protobuf.py @@ -1,6 +1,5 @@ from build.ab import Rule, Targets, emit, simplerule, filenamesof from build.utils import filenamesmatchingof, collectattrs -from build.c import cxxlibrary from types import SimpleNamespace from os.path import join import build.pkg # to get the protobuf package check @@ -26,7 +25,7 @@ def proto(self, name, srcs: Targets = [], deps: Targets = []): "$(PROTOC) --include_source_info --descriptor_set_out={outs[0]} {ins}" ], label="PROTO", - protosrcs=filenamesof(srcs), + args={"protosrcs": filenamesof(srcs)}, ) @@ -56,8 +55,45 @@ def protocc(self, name, srcs: Targets = [], deps: Targets = []): headers = {f[1:]: join(r.dir, f[1:]) for f in outs if f.endswith(".pb.h")} + from build.c import cxxlibrary + cxxlibrary( replaces=self, srcs=[r], + deps=deps, hdrs=headers, ) + + +@Rule +def protojava(self, name, srcs: Targets = [], deps: Targets = []): + outs = [] + + allsrcs = collectattrs(targets=srcs, name="protosrcs") + assert allsrcs, "no sources provided" + protos = [] + for f in filenamesmatchingof(allsrcs, "*.proto"): + protos += [f] + srcs += [f] + + r = simplerule( + name=f"{self.localname}_srcs", + cwd=self.cwd, + ins=protos, + outs=[f"={self.localname}.srcjar"], + deps=deps, + commands=[ + "mkdir -p {dir}/srcs", + "$(PROTOC) --java_out={dir}/srcs {ins}", + "$(JAR) cf {outs[0]} -C {dir}/srcs .", + ], + traits={"srcjar"}, + label="PROTOJAVA", + ) + + from build.java import javalibrary + + javalibrary( + replaces=self, + deps=[r] + deps, + ) diff --git a/build/utils.py b/build/utils.py index 13264242..ac57304e 100644 --- a/build/utils.py +++ b/build/utils.py @@ -1,5 +1,15 @@ -from build.ab import Rule, simplerule, Target, filenameof, Targets, filenamesof -from os.path import basename, splitext +from build.ab import ( + Rule, + Target, + Targets, + filenameof, + filenamesof, + cwdStack, + error, + simplerule, +) +from os.path import relpath, splitext, join, basename +from glob import glob import fnmatch import itertools @@ -23,6 +33,24 @@ def collectattrs(*, targets, name, initial=[]): return list(s) +def itemsof(pattern, root=None, cwd=None): + if not cwd: + cwd = cwdStack[-1] + if not root: + root = "." + + pattern = join(cwd, pattern) + root = join(cwd, root) + + result = {} + for f in glob(pattern, recursive=True): + try: + result[relpath(f, root)] = f + except ValueError: + error(f"file '{f}' is not in root '{root}'") + return result + + @Rule def objectify(self, name, src: Target, symbol): simplerule( @@ -48,7 +76,7 @@ def test( simplerule( replaces=self, ins=[command], - outs=["sentinel"], + outs=["=sentinel"], commands=["{ins[0]}", "touch {outs}"], deps=deps, label=label, @@ -57,7 +85,7 @@ def test( simplerule( replaces=self, ins=ins, - outs=["sentinel"], + outs=["=sentinel"], commands=commands + ["touch {outs}"], deps=deps, label=label, diff --git a/build/zip.py b/build/zip.py index e2b1bd13..421a5a10 100644 --- a/build/zip.py +++ b/build/zip.py @@ -15,7 +15,9 @@ @Rule -def zip(self, name, flags="", items: TargetsMap = {}): +def zip( + self, name, flags="", items: TargetsMap = {}, extension="zip", label="ZIP" +): cs = ["rm -f {outs[0]}"] ins = [] @@ -27,5 +29,9 @@ def zip(self, name, flags="", items: TargetsMap = {}): ins += [v] simplerule( - replaces=self, ins=ins, outs=[f"={name}.zip"], commands=cs, label="ZIP" + replaces=self, + ins=ins, + outs=[f"={self.localname}." + extension], + commands=cs, + label=label, )