Snapshot tests update (#1285)

* CJD library usage fix, use specific major/minor version.

Signed-off-by: Caroline Russell <[email protected]>

* Remove duplicate notification of missing file.

Signed-off-by: Caroline Russell <[email protected]>

---------

Signed-off-by: Caroline Russell <[email protected]>

.github/workflows/snapshot-tests.yml

@@ -42,7 +42,7 @@ jobs:
            python -m pip install --upgrade pip
            python -m pip install pytest
            git clone https://github.com/appthreat/cdxgen-samples.git /home/runner/work/original_snapshots
-           python -m pip install custom-json-diff
+           python -m pip install -r test/diff/requirements.txt
            curl -s "https://get.sdkman.io" | bash
            source "/home/runner/.sdkman/bin/sdkman-init.sh"
 

test/diff/README.md

@@ -12,7 +12,7 @@ before doing a bulk copy/paste. The html reports or the json diff can be used fo
 2. Download the zip of cdxgen boms generated for your PR from the [Test BOM Snapshots workflow](https://github.com/CycloneDX/cdxgen/actions/workflows/snapshot-tests.yml).
 3. Extract zips into separate directories.
 4. Create a python 3.10+ virtual environment.
-5. Install custom-json-diff with pip.
+5. Install custom-json-diff `pip install -r requirements.txt`
 6. Run diff_tests.py specifying the paths to the extracted snapshot directories.
    `diff_tests.py -d path/to/snapshots/from/step/1 path/to/snapshots/from/step/2`
 

test/diff/diff_tests.py

@@ -1,39 +1,43 @@
 import argparse
 import csv
 import json
+import logging
 import os
 
 from custom_json_diff.custom_diff import compare_dicts, perform_bom_diff, report_results
 from custom_json_diff.custom_diff_classes import Options
 
 
+logging.disable(logging.INFO)
+
+
 def build_args():
     parser = argparse.ArgumentParser()
     parser.add_argument(
         '--directories',
         '-d',
-        default=['/home/runner/work/original_snapshots','/home/runner/work/new_snapshots'],
+        default=['/home/runner/work/original_snapshots', '/home/runner/work/new_snapshots'],
         help='Directories containing the snapshots to compare',
         nargs=2
     )
     return parser.parse_args()
 
 
 def compare_snapshot(dir1, dir2, options, repo):
-    bom_1 = f"{dir1}/{repo["project"]}-bom.json"
-    bom_2 = f"{dir2}/{repo["project"]}-bom.json"
+    bom_1 = f"{dir1}/{repo['project']}-bom.json"
+    bom_2 = f"{dir2}/{repo['project']}-bom.json"
     options.file_1 = bom_1
     options.file_2 = bom_2
     options.output = f'{dir2}/{repo["project"]}-diff.json'
     if not os.path.exists(bom_1):
-        print(f'{bom_1} not found.')
         return f'{bom_1} not found.', f'{bom_1} not found.'
-    result, j1, j2 = compare_dicts(options)
-    if result != 0:
-        result_summary = perform_bom_diff(j1, j2)
-        report_results(result, result_summary, options, j1, j2)
-        return f"{repo['project']} failed.", result_summary
-    return None, None
+    status, j1, j2 = compare_dicts(options)
+    if status != 0:
+        status, result_summary = perform_bom_diff(j1, j2)
+        if status != 0:
+            report_results(status, result_summary, options, j1, j2)
+            return status, f"{repo['project']} failed.", result_summary
+    return status, None, None
 
 
 def perform_snapshot_tests(dir1, dir2):
@@ -48,9 +52,10 @@ def perform_snapshot_tests(dir1, dir2):
 
     failed_diffs = {}
     for repo in repo_data:
-        result, summary = compare_snapshot(dir1, dir2, options, repo)
+        status, result, summary = compare_snapshot(dir1, dir2, options, repo)
         if result:
             print(result)
+        if status != 0:
             failed_diffs[repo["project"]] = summary
 
     if failed_diffs:

test/diff/requirements.txt

@@ -0,0 +1 @@
+custom-json-diff~=1.5.5