From d451ca21a8ffcdc2a48ae76de8e9c826ebcccbe8 Mon Sep 17 00:00:00 2001 From: Henri Rosten Date: Mon, 11 Dec 2023 14:50:22 +0200 Subject: [PATCH] sbomnix: Remove command-line argument: --type - Remove `--type` command-line argument from sbomnix. Instead, change sbomnix so that it support `--buildtime` argument to align the command-line usage with other tools. - Update relevant documentation and tests Signed-off-by: Henri Rosten --- Makefile | 4 +-- README.md | 11 ++++--- src/nixupdate/nix_outdated.py | 10 +++--- src/sbomnix/main.py | 16 +++------ src/sbomnix/sbomdb.py | 42 +++++++++--------------- src/vulnxscan/vulnxscan_cli.py | 8 ++--- tests/compare_deps.py | 2 +- tests/test_sbomnix.py | 59 ++++++---------------------------- 8 files changed, 48 insertions(+), 104 deletions(-) diff --git a/Makefile b/Makefile index 478abee..75e0991 100644 --- a/Makefile +++ b/Makefile @@ -42,12 +42,12 @@ release-asset: clean ## Build release asset nix-shell -p nix-info --run "nix-info -m" nix-env -qa --meta --json -f $(shell nix-shell -p nix-info --run "nix-info -m" | grep "nixpkgs: " | cut -d'`' -f2) '.*' >meta.json mkdir -p build/ - nix run .#sbomnix -- result --type=runtime \ + nix run .#sbomnix -- result \ --meta=./meta.json \ --cdx=./build/sbom.runtime.cdx.json \ --spdx=./build/sbom.runtime.spdx.json \ --csv=./build/sbom.runtime.csv - nix run .#sbomnix -- result --type=buildtime \ + nix run .#sbomnix -- result --buildtime \ --meta=./meta.json \ --cdx=./build/sbom.buildtime.cdx.json \ --spdx=./build/sbom.buildtime.spdx.json \ diff --git a/README.md b/README.md index 505eb22..6b4af97 100644 --- a/README.md +++ b/README.md @@ -111,6 +111,9 @@ For reference, following is a link to graph from an example hello-world C progra +By default, all the tools in this repository assume runtime dependencies. This means, for instance, that unless specified otherwise, `sbomnix` will output an SBOM including the target runtime dependencies, `nixgraph` outputs runtime dependency graph, and `vulnxscan` and `nix_outdated` scan runtime dependencies. Since Nix needs to build the target output to determine the runtime dependencies, all the tools in this repository will also build (force-realise) the target output as part of each tool's invocation when determining the runtime dependencies. All the mentioned tools in this repository also support working with buildtime dependencies instead of runtime dependencies with the help of `--buildtime` command line argument. As mentioned earlier, generating buildtime dependencies in Nix does not require building the target. Similarly, when `--buildtime` is specified, the tools in this repository do not need to be build the given target. + + ## Usage Examples The usage examples work for both the built package, as well as inside the devshell. @@ -126,7 +129,7 @@ $ nix eval -f '' 'wget.outPath' #### Generate SBOM Based on Derivation File or Out-path By default `sbomnix` scans the given target and generates an SBOM including the runtime dependencies. -Keep in mind that determining the target runtime dependencies requires building the target. +Notice: determining the target runtime dependencies in Nix requires building the target. ```bash $ sbomnix /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 ... @@ -147,11 +150,11 @@ $ sbomnix /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 --meta meta.js ``` #### Generate SBOM Including Buildtime Dependencies -By default `sbomnix` scans the given target for runtime dependencies. You can tell sbomnix to determine the buildtime dependencies using the `--type` argument. -Acceptable values for `--type` are `runtime, buildtime, both`. Below example generates SBOM including buildtime dependencies. +By default `sbomnix` scans the given target for runtime dependencies. You can tell sbomnix to determine the buildtime dependencies using the `--buildtime` argument. +Below example generates SBOM including buildtime dependencies. Notice: as opposed to runtime dependencies, determining the buildtime dependencies does not require building the target. ```bash -$ sbomnix /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 --meta meta.json --type=buildtime +$ sbomnix /nix/store/8nbv1drmvh588pwiwsxa47iprzlgwx6j-wget-1.21.3 --meta meta.json --buildtime ``` #### Generate SBOM Based on Result Symlink `sbomnix` can be used with output paths too (e.g. anything which produces a result symlink): diff --git a/src/nixupdate/nix_outdated.py b/src/nixupdate/nix_outdated.py index 2ca55b6..dbbed91 100755 --- a/src/nixupdate/nix_outdated.py +++ b/src/nixupdate/nix_outdated.py @@ -75,9 +75,9 @@ def getargs(): ################################################################################ -def _generate_sbom(target_path, runtime=True, buildtime=False): +def _generate_sbom(target_path, buildtime=False): LOG.info("Generating SBOM for target '%s'", target_path) - sbomdb = SbomDb(target_path, runtime, buildtime, meta_path=None) + sbomdb = SbomDb(target_path, buildtime, meta_path=None) prefix = "nixdeps_" suffix = ".cdx.json" with NamedTemporaryFile(delete=False, prefix=prefix, suffix=suffix) as f: @@ -121,7 +121,7 @@ def _nix_visualize_csv_to_df(csvpath): # Followed by the version string r"(\d[-_.0-9pf]*g?b?(?:pre[0-9])*(?:\+git[0-9]*)?)" # Optionally followed by any of the following strings - r"(?:-lib|-bin|-env|-man|-su|-dev|-doc|-info|-nc|-host|-p[0-9]+|)" + r"(?:-lib|-bin|-env|-man|-su|-dev|-doc|-info|-nc|-host|-p[0-9]+|\.drv|)" # Followed by the end of line r"$" ) @@ -256,7 +256,7 @@ def main(): LOG.info("Checking %s dependencies referenced by '%s'", dtype, target_path_abs) exit_unless_nix_artifact(target_path_abs, force_realise=runtime) - sbom_path = _generate_sbom(target_path_abs, runtime, args.buildtime) + sbom_path = _generate_sbom(target_path_abs, args.buildtime) LOG.debug("Using SBOM '%s'", sbom_path) df_repology = _run_repology_cli(sbom_path) @@ -276,6 +276,8 @@ def main(): LOG.info("Not running nix-visualize due to '--buildtime' argument") df_nix_visualize = None + df_log(df_repology, logging.DEBUG) + df_log(df_nix_visualize, logging.DEBUG) df_report = _generate_report_df(df_nix_visualize, df_repology) _report(df_report, args) diff --git a/src/sbomnix/main.py b/src/sbomnix/main.py index d45c662..e5d7b2d 100755 --- a/src/sbomnix/main.py +++ b/src/sbomnix/main.py @@ -40,15 +40,8 @@ def getargs(): "to the output of this script (default: None)" ) parser.add_argument("--meta", nargs="?", help=helps, default=None) - helps = ( - "Set the type of dependencies included to the SBOM (default: runtime). " - "Note: generating 'runtime' SBOM requires realising (building) the " - "output paths of the target derivation. When 'runtime' SBOM is " - "requested, sbomnix will realise the target derivation unless its already " - "realised. See `nix-store --realise --help` for more info." - ) - types = ["runtime", "buildtime", "both"] - parser.add_argument("--type", choices=types, help=helps, default="runtime") + helps = "Scan buildtime dependencies instead of runtime dependencies" + parser.add_argument("--buildtime", help=helps, action="store_true") helps = ( "Set the depth of the included dependencies. As an example, --depth=1 " "indicates the SBOM should include only the NIX_PATH direct dependencies. " @@ -81,15 +74,14 @@ def main(): args = getargs() set_log_verbosity(args.verbose) target_path = args.NIX_PATH.resolve().as_posix() - runtime = args.type in ("runtime", "both") - buildtime = args.type in ("buildtime", "both") + runtime = args.buildtime is False exit_unless_nix_artifact(target_path, force_realise=runtime) if not args.meta: LOG.warning( "Command line argument '--meta' missing: SBOM will not include " "license information (see '--help' for more details)" ) - sbomdb = SbomDb(target_path, runtime, buildtime, args.meta, args.depth) + sbomdb = SbomDb(target_path, args.buildtime, args.meta, args.depth) if args.cdx: sbomdb.to_cdx(args.cdx) if args.spdx: diff --git a/src/sbomnix/sbomdb.py b/src/sbomnix/sbomdb.py index 725ecd1..829de44 100644 --- a/src/sbomnix/sbomdb.py +++ b/src/sbomnix/sbomdb.py @@ -4,7 +4,7 @@ # # SPDX-License-Identifier: Apache-2.0 -# pylint: disable=invalid-name, too-many-instance-attributes, too-many-arguments +# pylint: disable=invalid-name, too-many-instance-attributes """ Module for generating SBOMs in various formats """ @@ -27,19 +27,15 @@ class SbomDb: """Generates SBOMs in various formats""" - def __init__( - self, nix_path, runtime=True, buildtime=False, meta_path=None, depth=None - ): + def __init__(self, nix_path, buildtime=False, meta_path=None, depth=None): # self.uid specifies the attribute that SbomDb uses as unique # identifier for the sbom components. See the column names in # self.df_sbomdb (sbom.csv) for a list of all components' attributes. self.uid = "store_path" - self.runtime = runtime self.buildtime = buildtime self.meta_path = meta_path self.target_deriver = find_deriver(nix_path) - self.df_rdeps = None - self.df_bdeps = None + self.df_deps = None self.depth = depth self._init_dependencies(nix_path) self.df_sbomdb = None @@ -47,19 +43,17 @@ def __init__( self._init_sbomdb() self.uuid = uuid.uuid4() self.sbom_type = "runtime_and_buildtime" - if self.runtime and not self.buildtime: + if not self.buildtime: self.sbom_type = "runtime_only" - elif not self.runtime and self.buildtime: - self.sbom_type = "buildtime_only" def _init_dependencies(self, nix_path): - """Initialize runtime and buildtime dependencies (df_rdeps, df_bdeps)""" - if self.runtime: - runtime_dependencies = NixDependencies(nix_path, buildtime=False) - self.df_rdeps = self._get_dependencies_df(runtime_dependencies) + """Initialize dependencies (df_deps)""" if self.buildtime: buildtime_dependencies = NixDependencies(nix_path, buildtime=True) - self.df_bdeps = self._get_dependencies_df(buildtime_dependencies) + self.df_deps = self._get_dependencies_df(buildtime_dependencies) + else: + runtime_dependencies = NixDependencies(nix_path, buildtime=False) + self.df_deps = self._get_dependencies_df(runtime_dependencies) def _get_dependencies_df(self, nix_dependencies): if self.depth: @@ -75,18 +69,14 @@ def _get_dependencies_df(self, nix_dependencies): def _init_sbomdb(self): """Initialize self.df_sbomdb""" - if (self.df_bdeps is None or self.df_bdeps.empty) and ( - self.df_rdeps is None or self.df_rdeps.empty - ): + if self.df_deps is None or self.df_deps.empty: # No dependencies, so the only component in the sbom # will be the target itself paths = set([self.target_deriver]) else: - # Concat buildtime and runtime dependencies dropping duplicates - df_paths = pd.concat([self.df_rdeps, self.df_bdeps], ignore_index=True) # Get unique src_paths and target_paths - src_paths = df_paths["src_path"].unique().tolist() - target_paths = df_paths["target_path"].unique().tolist() + src_paths = self.df_deps["src_path"].unique().tolist() + target_paths = self.df_deps["target_path"].unique().tolist() paths = set(src_paths + target_paths) # Populate store based on the dependencies store = Store(self.buildtime) @@ -128,17 +118,17 @@ def _lookup_dependencies(self, drv, uid="store_path"): # Find runtime dependencies # Runtime dependencies: drv.outputs matches with target_path dfr = None - if self.df_rdeps is not None and not self.df_rdeps.empty: - df = self.df_rdeps[self.df_rdeps["target_path"].isin(drv.outputs)] + if self.df_deps is not None and not self.df_deps.empty: + df = self.df_deps[self.df_deps["target_path"].isin(drv.outputs)] # Find the requested 'uid' values for the dependencies (df.src_path) dfr = self.df_sbomdb_outputs_exploded.merge( df, how="inner", left_on=["outputs"], right_on=["src_path"] ).loc[:, [uid]] # Find buildtime dependencies dfb = None - if self.df_bdeps is not None and not self.df_bdeps.empty: + if self.df_deps is not None and not self.df_deps.empty: # Buildtime dependencies: drv.store_path matches with target_path - df = self.df_bdeps[self.df_bdeps["target_path"] == drv.store_path] + df = self.df_deps[self.df_deps["target_path"] == drv.store_path] # Find the requested 'uid' values for the dependencies (df.src_path) dfb = self.df_sbomdb.merge( df, how="inner", left_on=["store_path"], right_on=["src_path"] diff --git a/src/vulnxscan/vulnxscan_cli.py b/src/vulnxscan/vulnxscan_cli.py index 18d63f9..b0fd023 100755 --- a/src/vulnxscan/vulnxscan_cli.py +++ b/src/vulnxscan/vulnxscan_cli.py @@ -734,9 +734,9 @@ def _is_patched(row): return False -def _generate_sbom(target_path, runtime=True, buildtime=False): +def _generate_sbom(target_path, buildtime=False): LOG.info("Generating SBOM for target '%s'", target_path) - sbomdb = SbomDb(target_path, runtime, buildtime, meta_path=None) + sbomdb = SbomDb(target_path, buildtime, meta_path=None) prefix = "vulnxscan_" cdx_suffix = ".json" csv_suffix = ".csv" @@ -884,9 +884,7 @@ def main(): else: runtime = args.buildtime is False exit_unless_nix_artifact(target_path_abs, force_realise=runtime) - sbom_cdx_path, sbom_csv_path = _generate_sbom( - target_path_abs, runtime, args.buildtime - ) + sbom_cdx_path, sbom_csv_path = _generate_sbom(target_path_abs, args.buildtime) LOG.debug("Using cdx SBOM '%s'", sbom_cdx_path) LOG.debug("Using csv SBOM '%s'", sbom_csv_path) scanner.scan_vulnix(target_path_abs, args.buildtime) diff --git a/tests/compare_deps.py b/tests/compare_deps.py index 6a4586d..a6daf53 100755 --- a/tests/compare_deps.py +++ b/tests/compare_deps.py @@ -157,7 +157,7 @@ def compare_dependencies(df_sbom, df_graph, sbom_type, graph_type): df_sbom = df_sbom.astype(str) if (graph_type == "runtime" and sbom_type != "runtime_only") or ( - graph_type == "buildtime" and sbom_type != "buildtime_only" + graph_type == "buildtime" and sbom_type == "runtime_only" ): LOG.fatal("Unable to compare: graph='%s' vs sbom='%s'", graph_type, sbom_type) return False diff --git a/tests/test_sbomnix.py b/tests/test_sbomnix.py index 8f09007..80a39dc 100644 --- a/tests/test_sbomnix.py +++ b/tests/test_sbomnix.py @@ -100,7 +100,7 @@ def test_sbomnix_help(): def test_sbomnix_type_runtime(): - """Test sbomnix '--type=runtime' generates valid CycloneDX json""" + """Test sbomnix generates valid CycloneDX json with runtime dependencies""" out_path_cdx = TEST_WORK_DIR / "sbom_cdx_test.json" out_path_spdx = TEST_WORK_DIR / "sbom_spdx_test.json" _run_python_script( @@ -111,8 +111,6 @@ def test_sbomnix_type_runtime(): out_path_cdx.as_posix(), "--spdx", out_path_spdx.as_posix(), - "--type", - "runtime", ] ) assert out_path_cdx.exists() @@ -126,7 +124,7 @@ def test_sbomnix_type_runtime(): def test_sbomnix_type_buildtime(): - """Test sbomnix '--type=runtime' generates valid CycloneDX json""" + """Test sbomnix generates valid CycloneDX json with buildtime dependencies""" out_path_cdx = TEST_WORK_DIR / "sbom_cdx_test.json" out_path_spdx = TEST_WORK_DIR / "sbom_spdx_test.json" _run_python_script( @@ -137,34 +135,7 @@ def test_sbomnix_type_buildtime(): out_path_cdx.as_posix(), "--spdx", out_path_spdx.as_posix(), - "--type", - "buildtime", - ] - ) - assert out_path_cdx.exists() - assert out_path_spdx.exists() - cdx_schema_path = MYDIR / "resources" / "cdx_bom-1.3.schema.json" - assert cdx_schema_path.exists() - validate_json(out_path_cdx.as_posix(), cdx_schema_path) - spdx_schema_path = MYDIR / "resources" / "spdx_bom-2.3.schema.json" - assert spdx_schema_path.exists() - validate_json(out_path_spdx.as_posix(), spdx_schema_path) - - -def test_sbomnix_cdx_type_both(): - """Test sbomnix '--type=both' generates valid CycloneDX json""" - out_path_cdx = TEST_WORK_DIR / "sbom_cdx_test.json" - out_path_spdx = TEST_WORK_DIR / "sbom_spdx_test.json" - _run_python_script( - [ - SBOMNIX, - TEST_NIX_RESULT, - "--cdx", - out_path_cdx.as_posix(), - "--spdx", - out_path_spdx.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_cdx.exists() @@ -187,8 +158,6 @@ def test_sbomnix_depth(): TEST_NIX_RESULT, "--csv", out_path_csv_1.as_posix(), - "--type", - "runtime", ] ) assert out_path_csv_1.exists() @@ -201,8 +170,6 @@ def test_sbomnix_depth(): TEST_NIX_RESULT, "--csv", out_path_csv_2.as_posix(), - "--type", - "runtime", "--depth=1", ] ) @@ -319,8 +286,6 @@ def test_compare_deps_runtime(): TEST_NIX_RESULT, "--cdx", out_path_cdx.as_posix(), - "--type", - "runtime", ] ) assert out_path_cdx.exists() @@ -358,8 +323,7 @@ def test_compare_deps_buildtime(): TEST_NIX_RESULT, "--cdx", out_path_cdx.as_posix(), - "--type", - "buildtime", + "--buildtime", ] ) assert out_path_cdx.exists() @@ -384,8 +348,7 @@ def test_compare_subsequent_cdx_sboms(): TEST_NIX_RESULT, "--cdx", out_path_cdx_1.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_cdx_1.exists() @@ -397,8 +360,7 @@ def test_compare_subsequent_cdx_sboms(): TEST_NIX_RESULT, "--cdx", out_path_cdx_2.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_cdx_2.exists() @@ -421,8 +383,7 @@ def test_compare_subsequent_spdx_sboms(): TEST_NIX_RESULT, "--spdx", out_path_spdx_1.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_spdx_1.exists() @@ -434,8 +395,7 @@ def test_compare_subsequent_spdx_sboms(): TEST_NIX_RESULT, "--spdx", out_path_spdx_2.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_spdx_2.exists() @@ -461,8 +421,7 @@ def test_compare_spdx_and_cdx_sboms(): out_path_cdx.as_posix(), "--spdx", out_path_spdx.as_posix(), - "--type", - "both", + "--buildtime", ] ) assert out_path_cdx.exists()