Merge pull request #7 from robertvi/master

add the script to run validation tests against the SBML test suite

.github/workflows/non-omv.yml

@@ -49,6 +49,11 @@ jobs:
         cd SBML
         ./validateAll.sh
 
+    - name: Test test_suite output regeneration
+      run: |
+        cd test_suite
+        ./test_results_regeneration.sh
+
     - name: Final version info
       run: |
         omv list -V # list installed engines

test_suite/README.md

@@ -0,0 +1,19 @@
+# Tabulates results of validation tests on SBML test suite
+
+The `process_test_suite.py` script currently runs three tests from pyneuroml on all SBML files that match the input glob argument given to it: validate SBML, validate SBML (with strict unit checks activated), validate SEDML. The SEDML file is assumed to have the same name as the SBML file except the `.xml` extension must to changed to `-sedml.xml`. The output is a markdown file containing a simple table of the results obtained on each file, where each test result is recorded as either a `pass` or `FAIL`.
+
+- First download the [zipfile](https://github.com/sbmlteam/sbml-test-suite/releases/download/3.4.0/semantic_tests_with_sedml_and_graphs.v3.4.0.zip) (or the latest equivalent) of the [SBML test suite](https://github.com/sbmlteam/sbml-test-suite) that includes [SEDML](https://github.com/SED-ML/sed-ml) files.
+
+- Extract to any convenient location, for example using the `unzip` command:
+
+```
+unzip semantic_tests_with_sedml_and_graphs.v3.4.0.zip
+```
+
+- Then run the `process_test_suite.py` tool, pointing it towards the extracted test files, for example if the extraction folder is at `/home/vagrant/test_suite_files/semantic` and the `process_test_suite.py` script is in the current directory:
+
+```
+./process_test_suite.py --suite-path /home/vagrant/test_suite_files/semantic --suite-glob '*/*-sbml-l3v2.xml' --output-file results.md
+```
+
+- Use the `--help` option for more details or see the process script source code.

test_suite/process_test_suite.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+
+"""
+produce a markdown table of the results of running various tests on the SBML Test Suite
+
+get this version of the test suite that includes sedml versions or the sedml validation will fail:
+https://github.com/sbmlteam/sbml-test-suite/releases/download/3.4.0/semantic_tests_with_sedml_and_graphs.v3.4.0.zip
+"""
+
+import os
+import glob
+from pyneuroml.sbml import validate_sbml_files
+from pyneuroml.sedml import validate_sedml_files
+
+def parse_arguments():
+    "Parse command line arguments"
+
+    import argparse
+
+    parser = argparse.ArgumentParser(
+        description="Run various tests on the SBML Test Suite (or any similar set of SBML/SEDML files)"
+    )
+
+    parser.add_argument(
+        "--suite-path",
+        action="store",
+        type=str,
+        default=".",
+        help="Path to test suite directory, eg '~/repos/sbml-test-suite/cases/semantic'",
+    )
+
+    parser.add_argument(
+        "--suite-glob",
+        action="store",
+        type=str,
+        default="000*/*-sbml-l3v2.xml",
+        help="Shell-style glob matching test suite file(s) within suite_path, eg '000*/*-sbml-l3v2.xml'",
+    )
+
+    parser.add_argument(
+        "--suite-url-base",
+        action="store",
+        type=str,
+        default="https://github.com/sbmlteam/sbml-test-suite/blob/release/cases/semantic",
+        help="Base of the URL-to-suite-test-cases link to embed in results, use '' empty string to disable links",
+    )
+
+    parser.add_argument(
+        "--output-file",
+        action="store",
+        type=str,
+        default="results.md",
+        help="Path to file results will be written to, any parent directories must exist, eg ./results.md",
+    )
+
+    return parser.parse_args()
+
+#strings to use to represent passed and failed tests
+okay = "pass"
+fail = "FAIL"
+def pass_or_fail(result):
+    '''
+    convert True into "pass" and False into "fail"
+    as otherwise it's not obvious in the table what True and False mean
+    '''
+    global okay,fail
+
+    return okay if result else fail
+
+
+def add_case_url(case,fpath,url_base):
+    '''
+    insert URL link to original test case file online
+    effectively replaces args.suite_path with args.suite_url_base
+    this should produce a valid link for all the main intended use cases
+    of testing the sbml test suite using the default args
+    but will not handle all possible variations of globs and base directories
+    in which case it should be disabled by setting --suite-url-base=''
+    '''
+
+    url = os.path.join(url_base,fpath)
+    new_item = f'[{case}]({url})'
+    return new_item
+
+def process_cases(args):
+    """
+    process the test cases and write results out as a markdown table
+    with links to the test case files online (as noted above the sedml files are actually in a zip file)
+    with a summary of how many cases were tested and how many tests failed
+    """
+
+    header = "|case|valid-sbml|valid-sbml-units|valid-sedml|"
+    sep = "|---|---|---|---|"
+    summary="|cases={n_cases}|fails={n_failing[valid_sbml]}|fails={n_failing[valid_sbml_units]}|fails={n_failing[valid_sedml]}|"
+    row = "|{case}|{valid_sbml}|{valid_sbml_units}|{valid_sedml}|"
+
+    with open(args.output_file, "w") as fout:
+        #accumulate output in memory so we can put the nifty summary at the top
+        #instead of at the end of a very long table
+        output = []
+        output.append(header)
+        output.append(sep)
+        output.append("<results summary goes here>")
+        n_cases = 0
+        n_failing = {"valid_sbml":0, "valid_sbml_units":0, "valid_sedml":0 }
+
+        os.chdir(args.suite_path)
+        for fpath in sorted(glob.glob(args.suite_glob)):
+            sedml_path = fpath.replace(".xml", "-sedml.xml")
+            print(fpath)
+            assert os.path.isfile(fpath)
+            assert os.path.isfile(sedml_path)
+            case = os.path.basename(fpath)
+            if args.suite_url_base != '': case = add_case_url(case,fpath,args.suite_url_base)
+            valid_sbml = pass_or_fail(validate_sbml_files([fpath], strict_units=False))
+            valid_sbml_units = pass_or_fail(validate_sbml_files([fpath], strict_units=True))
+            valid_sedml = pass_or_fail(validate_sedml_files([sedml_path]))
+            output.append(row.format(**locals()))
+
+            #tally results so we can provide a summary
+            global okay,fail
+            n_cases +=1
+            if valid_sbml != okay: n_failing["valid_sbml"] += 1
+            if valid_sbml_units != okay: n_failing["valid_sbml_units"] += 1
+            if valid_sedml != okay: n_failing["valid_sedml"] += 1
+
+        output[2] = summary.format(**locals())
+        for line in output: fout.write(line+'\n')
+
+
+if __name__ == "__main__":
+    args = parse_arguments()
+
+    process_cases(args)