From 2926b2109a5bd5711d7465442b77f91f77dbe6fa Mon Sep 17 00:00:00 2001 From: "George V. Neville-Neil" Date: Wed, 2 Jun 2021 11:19:25 -0400 Subject: [PATCH 1/7] First draft of tagging and release commands. --- pycheribuild/projects/release.py | 147 +++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 pycheribuild/projects/release.py diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py new file mode 100644 index 000000000..fed6fd2d4 --- /dev/null +++ b/pycheribuild/projects/release.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2021 George V. Neville-Neil +# +# This software was developed by SRI International and the University of +# Cambridge Computer Laboratory (Department of Computer Science and +# Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of the +# DARPA SSITH research programme. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND +# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +# ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +# SUCH DAMAGE. +# +import tempfile +from pathlib import Path +from typing import Optional + +from pycheribuild.processutils import run_command +from pycheribuild.utils import default_make_jobs_count +from .cherisim import BuildBeriCtl, BuildCheriSim +from .project import CheriConfig, SimpleProject +from ..config.compilation_targets import CompilationTargets + + +class Tag(SimpleProject): + project_name = "tag" + + @classmethod + def setup_config_options(cls, **kwargs): + super().setup_config_options(**kwargs) + + cls.gittag = cls.add_config_option("gittag", help="Tag to apply") + + def __init__(self, config: CheriConfig): + super().__init__(config) + + def process(self): + source_root = self.config.source_root + repos = ["cheribuild", "cheribsd", "gdb", + "morello-llvm-project", "morello-qemu", + "morello-trusted-firmware-a"] + for repo in repos: + run_command("git", "-C", repo, "tag", self.gittag, cwd=source_root) + +class UnTag(SimpleProject): + project_name = "untag" + + @classmethod + def setup_config_options(cls, **kwargs): + super().setup_config_options(**kwargs) + + cls.gittag = cls.add_config_option("gittag", help="Tag to apply") + + def __init__(self, config: CheriConfig): + super().__init__(config) + + def process(self): + source_root = self.config.source_root + repos = ["cheribuild", "cheribsd", "gdb", + "morello-llvm-project", "morello-qemu", + "morello-trusted-firmware-a"] + for repo in repos: + run_command("git", "-C", repo, "tag", "-d", self.gittag, cwd=source_root) + +class Release(SimpleProject): + project_name = "release" + do_not_add_to_targets = True + +# dependencies = ["cheribsd-mfs-root-kernel-mips64-hybrid"] +# supported_architectures = [CompilationTargets.CHERIBSD_MIPS_HYBRID] + +# @classmethod +# def setup_config_options(cls, **kwargs): +# super().setup_config_options() +# print("SETUP") + + def __init__(self, config: CheriConfig): + super().__init__(config) + +class MorelloRelease(Release): + target = "morello-release" + dependencies = ["morello-llvm", "cheribsd-morello-purecap", + "gdb-native", + "gdb-morello-hybrid-for-purecap-rootfs", + "arm-none-eabi-toolchain", "morello-acpica", + "morello-scp-firmware", + "morello-trusted-firmware", + "morello-flash-images", + "disk-image-morello-purecap"] + def __init__(self, config: CheriConfig): + super().__init__(config) + install_script = Path(output_root, "install_and_run_fvp.sh") + install_script.write_text("""#!/bin/sh + dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + exec "${dir}/cheribuild.py" install-morello-fvp run-fvp-morello-purecap "$@""") + install_script.chmod(0o755) + + + def process(self): + output_root = self.config.output_root + run_command("bsdtar", "-cavf", output_root / "release.tar.xz", "-C", output_root, + "--options=xz:threads=" + str(default_make_jobs_count()), + "--options=compression-level=9", # reduce size a bit more + "morello-sdk/firmware", + "cheribsd-morello-purecap.img", + "sources/cheribuild", + "cheribuild.py", + cwd="/") + + run_command("sha256sum", output_root / "release.tar.xz") + +class RISCV64Release(Release): + target = "riscv64-release" + dependencies = ["gdb-native", "qemu", + "disk-image-riscv64-purecap"] + + def __init__(self, config: CheriConfig): + super().__init__(config) + + def process(self): + output_root = self.config.output_root + run_command("bsdtar", "-cavf", output_root / "release.tar.xz", "-C", output_root, + "--options=xz:threads=" + str(default_make_jobs_count()), + "--options=compression-level=9", # reduce size a bit more + "morello-sdk/firmware", + "cheribsd-morello-purecap.img", + "sources/cheribuild", + "cheribuild.py", + cwd="/") + + run_command("sha256sum", output_root / "release.tar.xz") From 2c310d704f2ebeb0d855973073e184e6b8003b1f Mon Sep 17 00:00:00 2001 From: "George V. Neville-Neil" Date: Fri, 4 Jun 2021 14:43:02 -0400 Subject: [PATCH 2/7] Make the repos list global. --- pycheribuild/projects/release.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py index fed6fd2d4..4dbce1b02 100644 --- a/pycheribuild/projects/release.py +++ b/pycheribuild/projects/release.py @@ -38,6 +38,10 @@ from ..config.compilation_targets import CompilationTargets +repos = ["cheribuild", "cheribsd", "gdb", + "morello-llvm-project", "morello-qemu", + "morello-trusted-firmware-a", "qemu"] + class Tag(SimpleProject): project_name = "tag" @@ -52,9 +56,6 @@ def __init__(self, config: CheriConfig): def process(self): source_root = self.config.source_root - repos = ["cheribuild", "cheribsd", "gdb", - "morello-llvm-project", "morello-qemu", - "morello-trusted-firmware-a"] for repo in repos: run_command("git", "-C", repo, "tag", self.gittag, cwd=source_root) @@ -72,9 +73,6 @@ def __init__(self, config: CheriConfig): def process(self): source_root = self.config.source_root - repos = ["cheribuild", "cheribsd", "gdb", - "morello-llvm-project", "morello-qemu", - "morello-trusted-firmware-a"] for repo in repos: run_command("git", "-C", repo, "tag", "-d", self.gittag, cwd=source_root) From 55433783f5984e7ed1f42ecf6ae0c4d3d51a13de Mon Sep 17 00:00:00 2001 From: George Neville-Neil Date: Tue, 8 Jun 2021 02:30:09 +0100 Subject: [PATCH 3/7] Clean up paths, get the right names for things. --- pycheribuild/projects/release.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py index 4dbce1b02..0afc1a81f 100644 --- a/pycheribuild/projects/release.py +++ b/pycheribuild/projects/release.py @@ -133,13 +133,16 @@ def __init__(self, config: CheriConfig): def process(self): output_root = self.config.output_root + source_root = self.config.source_root + + run_command("cp", "-a", source_root / "cheribuild", output_root) + run_command("bsdtar", "-cavf", output_root / "release.tar.xz", "-C", output_root, "--options=xz:threads=" + str(default_make_jobs_count()), "--options=compression-level=9", # reduce size a bit more - "morello-sdk/firmware", - "cheribsd-morello-purecap.img", - "sources/cheribuild", - "cheribuild.py", + "--exclude=*.git", + "cheribsd-riscv64-purecap.img", + "cheribuild", cwd="/") run_command("sha256sum", output_root / "release.tar.xz") From e12fdafd6b2f575ff67190f937aa55ae5e458e2d Mon Sep 17 00:00:00 2001 From: George Neville-Neil Date: Tue, 8 Jun 2021 23:31:47 +0100 Subject: [PATCH 4/7] Abrtract away more details to the parents classes. --- pycheribuild/projects/release.py | 42 +++++++++++++++++--------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py index 0afc1a81f..91a0e9f62 100644 --- a/pycheribuild/projects/release.py +++ b/pycheribuild/projects/release.py @@ -80,17 +80,17 @@ class Release(SimpleProject): project_name = "release" do_not_add_to_targets = True -# dependencies = ["cheribsd-mfs-root-kernel-mips64-hybrid"] -# supported_architectures = [CompilationTargets.CHERIBSD_MIPS_HYBRID] - -# @classmethod -# def setup_config_options(cls, **kwargs): -# super().setup_config_options() -# print("SETUP") - def __init__(self, config: CheriConfig): super().__init__(config) + def process(self): + output_root = self.config.output_root + source_root = self.config.source_root + + run_command("rm", "-rf", output_root / "cheribuild") + run_command("cp", "-a", source_root / "cheribuild", output_root) + + class MorelloRelease(Release): target = "morello-release" dependencies = ["morello-llvm", "cheribsd-morello-purecap", @@ -103,25 +103,29 @@ class MorelloRelease(Release): "disk-image-morello-purecap"] def __init__(self, config: CheriConfig): super().__init__(config) + + + def process(self): + super().process() + + output_root = self.config.output_root + install_script = Path(output_root, "install_and_run_fvp.sh") install_script.write_text("""#!/bin/sh dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) exec "${dir}/cheribuild.py" install-morello-fvp run-fvp-morello-purecap "$@""") install_script.chmod(0o755) - - def process(self): - output_root = self.config.output_root - run_command("bsdtar", "-cavf", output_root / "release.tar.xz", "-C", output_root, + run_command("bsdtar", "-cavf", output_root / "release-morello.tar.xz", "-C", output_root, "--options=xz:threads=" + str(default_make_jobs_count()), "--options=compression-level=9", # reduce size a bit more + "--exclude=*.git", "morello-sdk/firmware", "cheribsd-morello-purecap.img", - "sources/cheribuild", - "cheribuild.py", + "cheribuild", cwd="/") - run_command("sha256sum", output_root / "release.tar.xz") + run_command("sha256sum", output_root / "release-morello.tar.xz") class RISCV64Release(Release): target = "riscv64-release" @@ -132,12 +136,10 @@ def __init__(self, config: CheriConfig): super().__init__(config) def process(self): + super().process() output_root = self.config.output_root - source_root = self.config.source_root - - run_command("cp", "-a", source_root / "cheribuild", output_root) - run_command("bsdtar", "-cavf", output_root / "release.tar.xz", "-C", output_root, + run_command("bsdtar", "-cavf", output_root / "release-riscv64.tar.xz", "-C", output_root, "--options=xz:threads=" + str(default_make_jobs_count()), "--options=compression-level=9", # reduce size a bit more "--exclude=*.git", @@ -145,4 +147,4 @@ def process(self): "cheribuild", cwd="/") - run_command("sha256sum", output_root / "release.tar.xz") + run_command("sha256sum", output_root / "release-risc64.tar.xz") From 7701f23cdc516f31af2d4fce4b792825d957c484 Mon Sep 17 00:00:00 2001 From: George Neville-Neil Date: Wed, 9 Jun 2021 17:48:50 +0100 Subject: [PATCH 5/7] First working version of the RISCV-64 release target. --- pycheribuild/projects/release.py | 36 +++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py index 91a0e9f62..c60dff2d9 100644 --- a/pycheribuild/projects/release.py +++ b/pycheribuild/projects/release.py @@ -84,11 +84,15 @@ def __init__(self, config: CheriConfig): super().__init__(config) def process(self): - output_root = self.config.output_root source_root = self.config.source_root - run_command("rm", "-rf", output_root / "cheribuild") - run_command("cp", "-a", source_root / "cheribuild", output_root) + Path(source_root, "cheribuild/cheribuild.json").write_text("""{ + "source-root": "../../sources", + "build-root": "../../build", + "output-root": "../../output", + "skip-update": true +} + """) class MorelloRelease(Release): @@ -108,6 +112,7 @@ def __init__(self, config: CheriConfig): def process(self): super().process() + root = self.config.output_root.parent output_root = self.config.output_root install_script = Path(output_root, "install_and_run_fvp.sh") @@ -116,13 +121,15 @@ def process(self): exec "${dir}/cheribuild.py" install-morello-fvp run-fvp-morello-purecap "$@""") install_script.chmod(0o755) - run_command("bsdtar", "-cavf", output_root / "release-morello.tar.xz", "-C", output_root, + run_command("bsdtar", "-cavf", output_root / "release-morello.tar.xz", + "-C", root, "--options=xz:threads=" + str(default_make_jobs_count()), "--options=compression-level=9", # reduce size a bit more "--exclude=*.git", - "morello-sdk/firmware", - "cheribsd-morello-purecap.img", - "cheribuild", + "output/morello-sdk/firmware", + "output/cheribsd-morello-purecap.img", + "output/sdk", + "sources/cheribuild", cwd="/") run_command("sha256sum", output_root / "release-morello.tar.xz") @@ -130,6 +137,7 @@ def process(self): class RISCV64Release(Release): target = "riscv64-release" dependencies = ["gdb-native", "qemu", + "bbl-baremetal-riscv64-purecap", "disk-image-riscv64-purecap"] def __init__(self, config: CheriConfig): @@ -137,14 +145,18 @@ def __init__(self, config: CheriConfig): def process(self): super().process() + root = self.config.output_root.parent output_root = self.config.output_root - run_command("bsdtar", "-cavf", output_root / "release-riscv64.tar.xz", "-C", output_root, + run_command("bsdtar", "-cavf", output_root / "release-riscv64.tar.xz", + "-C", root, "--options=xz:threads=" + str(default_make_jobs_count()), - "--options=compression-level=9", # reduce size a bit more + #"--options=compression-level=9", # reduce size a bit more "--exclude=*.git", - "cheribsd-riscv64-purecap.img", - "cheribuild", + "output/cheribsd-riscv64-purecap.img", + "output/rootfs-riscv64-purecap", + "output/sdk", + "sources/cheribuild", cwd="/") - run_command("sha256sum", output_root / "release-risc64.tar.xz") + run_command("sha256sum", output_root / "release-riscv64.tar.xz") From f60e63505ffa45297bb112d59ced66906bb78775 Mon Sep 17 00:00:00 2001 From: George Neville-Neil Date: Thu, 24 Jun 2021 17:47:15 +0100 Subject: [PATCH 6/7] Address updates to project_name and target --- pycheribuild/projects/release.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pycheribuild/projects/release.py b/pycheribuild/projects/release.py index c60dff2d9..0ab88b343 100644 --- a/pycheribuild/projects/release.py +++ b/pycheribuild/projects/release.py @@ -43,7 +43,7 @@ "morello-trusted-firmware-a", "qemu"] class Tag(SimpleProject): - project_name = "tag" + target = "tag" @classmethod def setup_config_options(cls, **kwargs): @@ -60,8 +60,8 @@ def process(self): run_command("git", "-C", repo, "tag", self.gittag, cwd=source_root) class UnTag(SimpleProject): - project_name = "untag" - + target = "untag" + @classmethod def setup_config_options(cls, **kwargs): super().setup_config_options(**kwargs) From 3133912492c94e5b53259ab9e601769c9963aac2 Mon Sep 17 00:00:00 2001 From: "George V. Neville-Neil" Date: Thu, 12 Aug 2021 12:12:59 -0400 Subject: [PATCH 7/7] Update to latest Ubuntu and tools. Add UID/GID support for running Docker. --- docker/Dockerfile | 16 +++++++++++----- pycheribuild/__main__.py | 8 ++++++-- pycheribuild/config/chericonfig.py | 2 ++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8d63c0860..2b1e388e9 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,9 @@ -FROM ubuntu:18.04 +FROM ubuntu:20.04 LABEL maintainer="Alexander.Richardson@cl.cam.ac.uk" +ARG DEBIAN_FRONTEND=noninteractive + RUN apt-get update && apt-get install -y \ make ninja-build \ gcc g++ \ @@ -9,7 +11,9 @@ RUN apt-get update && apt-get install -y \ python3-minimal \ lsb-release \ wget \ - samba + samba \ + texlive-base \ + texinfo # RUN git config --global http.sslVerify false # RUN cd /tmp && git clone https://github.com/arichardson/bmake && cd bmake \ @@ -20,18 +24,20 @@ COPY cheribuild.json /root/.config/cheribuild.json # deps to build QEMU+elftoolchain: RUN apt-get update && apt-get install -y \ - libtool pkg-config python-minimal autotools-dev automake autoconf libglib2.0-dev libpixman-1-dev \ + libtool pkg-config python2-minimal autotools-dev automake autoconf libglib2.0-dev libpixman-1-dev \ bison groff-base libarchive-dev flex -# INSTALL clang 8.0 +# INSTALL clang 10.0 RUN wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - RUN echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-8 main" > /etc/apt/sources.list.d/llvm.list # RUN echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu trusty main" > /etc/apt/sources.list.d/r-toolchain.list # RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F -RUN apt-get update && apt-get install -y clang-8 lld-8 +RUN apt-get update && apt-get install -y clang-10 lld-10 RUN apt-get update && apt-get install -y cmake +RUN groupadd -g 12345 -r cheri && useradd -u 12345 --no-log-init -r -g cheri cheri + VOLUME ["/cheribuild", "/source", "/build", "/output"] ENV PATH /cheribuild:$PATH CMD bash diff --git a/pycheribuild/__main__.py b/pycheribuild/__main__.py index 7b374f46c..e5df0ad57 100644 --- a/pycheribuild/__main__.py +++ b/pycheribuild/__main__.py @@ -198,8 +198,12 @@ def real_main(): subprocess.check_call(start_cmd) docker_run_cmd = ["docker", "exec", cheri_config.docker_container] + cheribuild_args else: - docker_run_cmd = ["docker", "run", "--user", str(os.getuid()) + ":" + str(os.getgid()), - "--rm", "--interactive", "--tty"] + docker_dir_mappings + if cheri_config.docker_uid != "": + docker_run_cmd = ["docker", "run", "--user", cheri_config.docker_uid + ":" + cheri_config.docker_gid, + "--rm", "--interactive", "--tty"] + docker_dir_mappings + else: + docker_run_cmd = ["docker", "run", "--user", str(os.getuid()) + ":" + str(os.getgid()), + "--rm", "--interactive", "--tty"] + docker_dir_mappings docker_run_cmd += [cheri_config.docker_container] + cheribuild_args run_command(docker_run_cmd, config=cheri_config, give_tty_control=True) except subprocess.CalledProcessError as e: diff --git a/pycheribuild/config/chericonfig.py b/pycheribuild/config/chericonfig.py index 0e5b29e6d..05d675ac9 100644 --- a/pycheribuild/config/chericonfig.py +++ b/pycheribuild/config/chericonfig.py @@ -278,6 +278,8 @@ def __init__(self, loader, action_class): help="Attach to the same container again (note: " "docker-container option must be an id rather than " "a container name") + self.docker_uid = loader.add_option("docker-uid", help="UID to under Docker", default="", group=loader.docker_group) + self.docker_gid = loader.add_option("docker-gid", help="GID to under Docker", default="", group=loader.docker_group) # compilation db options: self.create_compilation_db = loader.add_commandline_only_bool_option(