diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..05ce33b182 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,95 @@ +name: CI + +on: + push: + branches: + - main # push commit to the main branch + tags: + - 'v2*' # push tag starting with "v2" to the main branch + pull_request: + branches: + - main # pull request to the main branch + workflow_dispatch: # allow manual triggering + +defaults: + run: + shell: bash + +jobs: + build-linux: + name: CI py${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10'] + max-parallel: 5 + env: + python_version: ${{ matrix.python-version }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Deploy Information + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + run: | + echo "The HTML NeXus User Manual will be pushed to" + echo " https://github.com/nexusformat/definitions/tree/gh-pages" + echo "The HTML NeXus User Manual will be deployed on" + echo " https://nexusformat.github.io/definitions/" + + - name: Install Requirements + run: | + python3 -m pip install --upgrade pip setuptools + make install + python3 -m pip list + + - name: Check Code Style + run: | + make style + + - name: Run Tests + run: | + make test + + - name: Install LaTeX + run: | + sudo apt-get update -y && \ + sudo apt-get install -y \ + latexmk \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-fonts-recommended + + - name: Generate build files + run: | + make prepare + + - name: Build Impatient Guid + run: | + make impatient-guide + ls -lAFgh build/impatient-guide/build/html/index.html + ls -lAFgh build/impatient-guide/build/latex/NXImpatient.pdf + + - name: Build User Manual + run: | + make pdf + make html + ls -lAFgh build/manual/build/html/index.html + ls -lAFgh build/manual/build/latex/nexus.pdf + + - name: Build and Commit the User Manual + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + uses: sphinx-notes/pages@master + with: + # path to the conf.py directory + documentation_path: build/manual/source + + - name: Deploy the User Manual + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: gh-pages diff --git a/.github/workflows/publish-sphinx.yml b/.github/workflows/publish-sphinx.yml deleted file mode 100644 index 7ae1b7fdf1..0000000000 --- a/.github/workflows/publish-sphinx.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Publish Sphinx Docs to GitHub Pages - -on: - # Triggers the workflow on push events but only for the main branch - push: - branches: - - main - tags: - - 'v2*' # all tags begining with v2 - -# see: https://sphinx-notes.github.io/pages/ -# see: https://github.com/marketplace/actions/sphinx-to-github-pages - -jobs: - - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@master - with: - fetch-depth: 0 # otherwise, you will fail to push refs to dest repo - - - name: Install build requirements - run: | - pip install -r requirements.txt - - - name: Diagnostic - run: | - pip list - - - name: Run the test suite - run: | - make test - - - name: Prepare for out-of-source Sphinx build - run: | - make prepare - - - name: Diagnostic - run: | - ls -lAFGh - ls -lAFGh ./build - - - name: Install LaTeX - run: | - sudo apt-get update -y && \ - sudo apt-get install -y \ - latexmk \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended - - - name: Build impatient guide - run: | - make -C build impatient-guide - ls -lAFGh ./build/impatient-guide/_build/latex/*.pdf - - # Copy to documentation source directory - cp \ - ./build/impatient-guide/_build/latex/NXImpatient.pdf \ - ./build/manual/source/_static/ - - - name: Build PDF of manual - run: | - make -C build nxdl2rst pdf - - # Copy to documentation source directory - cp \ - ./build/manual/build/latex/nexus.pdf \ - ./build/manual/source/_static/NeXusManual.pdf - - - name: Include other PDFs - run: | - wget https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf -O ./build/manual/source/_static/NeXusIntern.pdf - wget https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf -O ./build/manual/source/_static/NXtranslate.pdf - - - name: Build (html) and Commit - uses: sphinx-notes/pages@master - with: - # path to conf.py directory - documentation_path: build/manual/source - - - name: Publish if refs/tags - # remove/comment next line to push right away - if: startsWith(github.ref, 'refs/tags') - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages diff --git a/.github/workflows/syntax-checks.yml b/.github/workflows/syntax-checks.yml deleted file mode 100644 index 62cbf25f79..0000000000 --- a/.github/workflows/syntax-checks.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Syntax Checks - -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build-linux: - name: CI py${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - max-parallel: 5 - - steps: - - uses: actions/checkout@v2 - - - name: Install build requirements - run: | - pip install -r requirements.txt - - - name: Diagnostics - shell: bash -l {0} - run: | - pwd - ls -lart - - pip list - printenv | sort - - - name: Run tests - shell: bash -l {0} - run: | - make test - - - name: Build the documentation - shell: bash -l {0} - run: | - make prepare - make -C build nxdl2rst html - ls -lAGh diff --git a/.gitignore b/.gitignore index 5f82132a98..ff21c1627c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,18 +7,8 @@ __pycache__/ # Build artifacts build/ -_build/ makelog.txt -# Obsolete in-source build artifacts -manual/pdf -manual/source/nexus.pdf -manual/source/nxdl_desc.rst -manual/source/*.table -manual/source/classes/*/NX*.rst -manual/source/classes/*/*.nxdl.xml -manual/source/classes/contributed_definitions/canSAS - # Unknown /python/ __github_creds__.txt diff --git a/BUILDING.rst b/BUILDING.rst deleted file mode 100644 index 63e1b5aa86..0000000000 --- a/BUILDING.rst +++ /dev/null @@ -1,82 +0,0 @@ -The documentation relies on Sphinx (a Python package) for its organization. The -GNU ``make`` program and Python are used to build the NeXus documentation. The -default build assembles the HTML version. You have the choice to build the -documentation in two places: - -* in the source tree -* out of the source tree - -Out-of-Tree documentation -========================= - -There are two ways to build out-of-source. - -Outside the source tree ------------------------ - -To build the NeXus documentation outside the -source tree, - -#. create the target directory for the documentation to be built:: - - mkdir /some/where/else - -#. note the definitions source directory - (the directory where this README file is located):: - - export SOURCE_DIR=/path/to/nexus/definitions - -#. copy the source to the target using this NeXus Python tool:: - - cd /some/where/else - python $(SOURCE_DIR)/utils/build_preparation.py $(SOURCE_DIR) - -#. build the documentation:: - - make clean - make - -The HTML documentation is located in this folder:: - - /some/where/else/manual/build/html/ - - -Inside the source tree, in a temporary directory ------------------------------------------------- - -Alternatively, as is a common practice with `cmake `_, -you can build *out-of-source* (sort of) in a temporary -``$(SOURCE_DIR)/build`` directory. For this, the *Makefile* -has the *builddir* rule:: - - export SOURCE_DIR=/path/to/nexus/definitions - cd $(SOURCE_DIR) - make builddir - cd build - make clean - make - -This is all done with one make command:: - - export SOURCE_DIR=/path/to/nexus/definitions - cd $(SOURCE_DIR) - make makebuilddir - -The HTML documentation is located in this folder:: - - $(SOURCE_DIR)/build/manual/build/html/ - -In-Tree documentation -===================== - -To build the NeXus documentation within the -source tree, go to the root directory -(the directory where this README file is located), -and type:: - - make clean - make - -The HTML documentation is located in this folder:: - - ./manual/build/html/ diff --git a/Makefile b/Makefile index c4bd30b99d..ae556d7339 100644 --- a/Makefile +++ b/Makefile @@ -3,109 +3,86 @@ # purpose: # build resources in NeXus definitions tree -SUBDIRS = manual impatient-guide PYTHON = python3 -DIR_NAME = "$(shell basename $(realpath .))" +SPHINX = sphinx-build BUILD_DIR = "build" -.PHONY: $(SUBDIRS) all clean html pdf nxdl2rst prepare test local help +.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local help :: @echo "" @echo "NeXus: Testing the NXDL files and building the documentation:" @echo "" - @echo "make all -C . Total (re)build of the manual, starting from root directory." - @echo "make all -C build Builds complete web site for the manual (in build directory)." - @echo "make clean Remove build products from some directories." - @echo "make impatient-guide -C build Build html & PDF versions of the Guide for the Impatient." - @echo "make html -C build Build HTML version of manual, requires nxdl2rst first," - @echo "make nxdl2rst -C build Document each NXDL class, add to manual source." - @echo "make pdf -C build Build PDF version of manual, requires nxdl2rst first" - @echo "make prepare -C . (Re)create the build directory." - @echo "make test Apply all Python-coded tests." - @echo "make local -C . (Developer use) Build nxdl2rst and the html manual." + + @echo "make install Install all requirements to run tests and builds." + @echo "make style Check python coding style." + @echo "make autoformat Format all files to the coding style conventions." + @echo "make test Run NXDL syntax and documentation tests." + @echo "make clean Remove all build files." + @echo "make prepare (Re)create all build files." + @echo "make html Build HTML version of manual. Requires prepare first." + @echo "make pdf Build PDF version of manual. Requires prepare first." + @echo "make impatient-guide Build html & PDF versions of the Guide for the Impatient. Requires prepare first." + @echo "make all Builds complete web site for the manual (in build directory)." + @echo "make local (Developer use) Test, prepare and build the HTML manual." @echo "" @echo "Note: All builds of the manual will occur in the 'build/' directory." @echo " For a complete build, run 'make all' in the root directory." - @echo " To delete and then create the 'build/' directory, run 'make prepare' in the root directory." - @echo " For syntax checks of the NXDL files, run 'make test' in either root or 'build/' directory." - @echo " Developers might find it easier to 'make local' to confirm the documentation builds." + @echo " Developers of the NeXus class definitions can use 'make local' to" + @echo " confirm the documentation builds." @echo "" -all :: -ifneq ($(DIR_NAME), $(BUILD_DIR)) - # root directory - $(MAKE) test - $(MAKE) prepare - $(MAKE) -C $(BUILD_DIR) all -else - # root/build directory - $(MAKE) clean - $(MAKE) impatient-guide - $(MAKE) nxdl2rst - $(MAKE) html - $(MAKE) pdf +install :: + $(PYTHON) -m pip install -r requirements.txt - cp impatient-guide/_build/latex/NXImpatient.pdf manual/build/html/_static/NXImpatient.pdf - cp manual/build/latex/nexus.pdf manual/build/html/_static/NeXusManual.pdf +style :: + $(PYTHON) -m black --check dev_tools + $(PYTHON) -m flake8 dev_tools + $(PYTHON) -m isort --check dev_tools - @echo "HTML built: `ls -lAFgh manual/build/html/index.html`" - @echo "PDF built: `ls -lAFgh manual/build/latex/nexus.pdf`" -endif +autoformat :: + $(PYTHON) -m black dev_tools + $(PYTHON) -m isort dev_tools -clean :: - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done - -impatient-guide :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) html -C $@ - $(MAKE) latexpdf -C $@ -endif +test :: + $(PYTHON) -m pytest dev_tools -html :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) html -C manual -endif +clean :: + $(RM) -rf $(BUILD_DIR) -nxdl2rst :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) -C manual/source PYTHON=$(PYTHON) -endif +prepare :: + $(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR) + $(PYTHON) -m dev_tools impatient --prepare --build-root $(BUILD_DIR) pdf :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - # expect pass to fail (thus exit 0) since nexus.ind not found first time - # extra option needed to satisfy "levels nested too deeply" error - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) - - # create the missing file - makeindex -s manual/build/latex/python.ist manual/build/latex/nexus.idx + $(SPHINX) -M latexpdf $(BUILD_DIR)/manual/source/ $(BUILD_DIR)/manual/build + cp $(BUILD_DIR)/manual/build/latex/nexus.pdf $(BUILD_DIR)/manual/source/_static/NeXusManual.pdf - # second pass will also fail but we can ignore it without problem - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) - - # third time should be no errors - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) -endif - -prepare :: -ifneq ($(DIR_NAME), $(BUILD_DIR)) - $(RM) -rf $(BUILD_DIR) - mkdir -p $(BUILD_DIR) - $(PYTHON) utils/build_preparation.py . $(BUILD_DIR) -endif +html :: + $(SPHINX) -b html -W $(BUILD_DIR)/manual/source/ $(BUILD_DIR)/manual/build/html -test :: - $(PYTHON) utils/test_suite.py +impatient-guide :: + $(SPHINX) -b html -W $(BUILD_DIR)/impatient-guide/ $(BUILD_DIR)/impatient-guide/build/html + $(SPHINX) -M latexpdf $(BUILD_DIR)/impatient-guide/ $(BUILD_DIR)/impatient-guide/build + cp $(BUILD_DIR)/impatient-guide/build/latex/NXImpatient.pdf $(BUILD_DIR)/manual/source/_static/NXImpatient.pdf # for developer's use on local build host local :: $(MAKE) test $(MAKE) prepare - $(MAKE) nxdl2rst -C $(BUILD_DIR) - $(MAKE) html -C $(BUILD_DIR) + $(MAKE) html + +all :: + $(MAKE) clean + $(MAKE) prepare + $(MAKE) impatient-guide + $(MAKE) pdf + $(MAKE) html + @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/impatient-guide/build/html/index.html`" + @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/impatient-guide/build/latex/NXImpatient.pdf`" + @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`" + @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`" + # NeXus - Neutron and X-ray Common Data Format # diff --git a/README.md b/README.md index bb79e5b1f5..6bb56e3ea5 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,35 @@ * Documentation: https://manual.nexusformat.org/ * Release Notes: https://github.com/nexusformat/definitions/wiki/Release-Notes * License: [![License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) -* Continuous Integration: [![Syntax Checking](https://github.com/nexusformat/definitions/actions/workflows/syntax-checks.yml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/syntax-checks.yml) -* Continuous Deployment: [![Publish Documentation](https://github.com/nexusformat/definitions/actions/workflows/publish-sphinx.yml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/publish-sphinx.yml) +* Test, Build and Deploy: [![CI](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml) + +## NeXus definition developers + +After making a change to the NeXus class definitions there are two important checks +to be made before commiting the change: + + 1. check whether the change does not violate any syntax rules + 2. verify whether the change looks as intended in the HTML documentation + +First install the test and build requirements with this command (only run once) + + make install + +Then verify syntax and build the HTML manual with this command + + make local + +Open the HTML manual in a web brower for visual verification + + firefox build/manual/build/html/index.html + +## Repository content These are the components that define the structure of NeXus data files in the development directory. component | description -------------------------------|------------------------ -[BUILDING.rst](BUILDING.rst) | how to build the documentation [CHANGES.rst](CHANGES.rst) | Change History [LGPL.txt](LGPL.txt) | one proposed license model [NXDL_VERSION](NXDL_VERSION) | the current NXDL version number @@ -29,3 +49,4 @@ package/ | directory for packaging this content utils/ | various tools used in the definitions tree www/ | launch (home) page of NeXus WWW site xslt/ | various XML stylesheet transformations +dev_tools/ | developer tools for testing and building \ No newline at end of file diff --git a/dev_tools/README.md b/dev_tools/README.md new file mode 100644 index 0000000000..2ebdaacdf0 --- /dev/null +++ b/dev_tools/README.md @@ -0,0 +1,101 @@ +# NeXus development tools + +The NeXus development tools are used to validate NeXus class definitions +and generate all files needed for documentation building with sphinx. + +## Quick reference + +Install all requirements + +```bash +python3 -m pip install -r requirements.txt +``` + +Run all tests + +```bash +pytest dev_tools +``` + +Generate the build files + +```bash +python3 -m dev_tools impatient --prepare +python3 -m dev_tools manual --prepare +``` + +Build the PDF manuals + +```bash +sphinx-build -M latexpdf build/impatient-guide/ build/impatient-guide/build +cp build/impatient-guide/build/latex/NXImpatient.pdf build/manual/source/_static/NXImpatient.pdf + +sphinx-build -M latexpdf build/manual/source/ build/manual/build +cp build/manual/build/latex/nexus.pdf build/manual/source/_static/NeXusManual.pdf +``` + +Build the HTML manuals + +```bash +sphinx-build -b html -W build/impatient-guide/ build/impatient-guide/build/html +sphinx-build -b html -W build/manual/source/ build/manual/build/html +``` + +Auto-formatting and syntax checking to the developer tools themselves + +```bash +black dev_tools +isort dev_tools +flake8 dev_tools +``` + +## Prepare environment (optional) + +Create a fresh python environment + +```bash +python3 -m venv nexusenv +``` + +Activate the environment on Linux or MacOS + +```bash +source nexusenv/bin/activate +``` + +Activate the environment on Windows + +```bash +nexusenv\Scripts\activate.bat +``` + +## Command line interface + +```bash +python3 -m dev_tools --help +``` + +### Single NeXus class definition + +This subcommand provides syntax checking and documentation building +(print, save and diff) for a single NeXus class definition + +```bash +python3 -m dev_tools nxclass --help +``` + +For example the difference between the existing documentation of +a NeXus class definition and the generated documentation + +```bash +python3 -m dev_tools nxclass nxaperture --diff +--- build + ++++ source + +@@ -1,4 +1,4 @@ + +-.. auto-generated by script ../../../../utils/nxdl2rst.py from the NXDL source NXaperture.nxdl.xml + ++.. auto-generated by dev_tools.docs.rst from the NXDL source base_classes/NXaperture.nxdl.xml +``` diff --git a/dev_tools/__init__.py b/dev_tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/__main__.py b/dev_tools/__main__.py new file mode 100644 index 0000000000..6e728a8e0a --- /dev/null +++ b/dev_tools/__main__.py @@ -0,0 +1,53 @@ +import argparse +import sys + +from .apps import dir_app +from .apps import impatient_app +from .apps import manual_app +from .apps import nxclass_app + + +def main(argv=None): + parser = argparse.ArgumentParser(description="NeXus development tools") + + subparsers = parser.add_subparsers(help="Commands", dest="command") + + nxclass_parser = subparsers.add_parser( + "nxclass", help="Test and documentation for a single NeXus class" + ) + nxclass_app.nxclass_args(nxclass_parser) + dir_app.dir_args(nxclass_parser) + + manual_parser = subparsers.add_parser( + "manual", help="Test and prepare User Manual building" + ) + manual_app.manual_args(manual_parser) + dir_app.dir_args(manual_parser) + + impatient_parser = subparsers.add_parser( + "impatient", help="Prepare Impatient Guide building" + ) + impatient_app.impatient_args(impatient_parser) + dir_app.dir_args(impatient_parser) + + if argv is None: + argv = sys.argv + args = parser.parse_args(argv[1:]) + + if args.command == "nxclass": + dir_app.dir_exec(args) + nxclass_app.nxclass_exec(args) + elif args.command == "manual": + dir_app.dir_exec(args) + manual_app.manual_exec(args) + elif args.command == "impatient": + dir_app.dir_exec(args) + impatient_app.impatient_exec(args) + else: + parser.print_help() + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev_tools/apps/__init__.py b/dev_tools/apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/apps/dir_app.py b/dev_tools/apps/dir_app.py new file mode 100644 index 0000000000..ddc04b10e9 --- /dev/null +++ b/dev_tools/apps/dir_app.py @@ -0,0 +1,31 @@ +from ..globals import directories + + +def dir_args(parser): + parser.add_argument( + "--source-root", + type=str, + default=None, + help="Source root directory" f" Default: {directories.get_source_root()}", + ) + parser.add_argument( + "--nxdl-root", + type=str, + default=None, + help="NXDL root directory" f" Default: {directories.get_nxdl_root()}", + ) + parser.add_argument( + "--build-root", + type=str, + default=None, + help="Build root directory" f" Default: {directories.get_build_root()}", + ) + + +def dir_exec(args): + if args.source_root: + directories.set_source_root(args.source_root) + if args.nxdl_root: + directories.set_nxdl_root(args.nxdl_root) + if args.build_root: + directories.set_build_root(args.build_root) diff --git a/dev_tools/apps/impatient_app.py b/dev_tools/apps/impatient_app.py new file mode 100644 index 0000000000..b915435820 --- /dev/null +++ b/dev_tools/apps/impatient_app.py @@ -0,0 +1,15 @@ +from ..globals import directories +from ..utils.copy import copydir + + +def impatient_args(parser): + parser.add_argument( + "--prepare", + action="store_true", + help="Create the build files for the NeXus Impatient Guide", + ) + + +def impatient_exec(args): + if args.prepare: + copydir(directories.impatient_source_root(), directories.impatient_build_root()) diff --git a/dev_tools/apps/manual_app.py b/dev_tools/apps/manual_app.py new file mode 100644 index 0000000000..768332b58d --- /dev/null +++ b/dev_tools/apps/manual_app.py @@ -0,0 +1,138 @@ +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..docs import XSDDocGenerator +from ..docs.nxdl_index import nxdl_indices +from ..docs.xsd_units import generate_xsd_units_doc +from ..globals import directories +from ..nxdl import iter_definitions +from ..nxdl import validate_definition +from ..utils.copy import copy_files +from ..utils.copy import copydir +from ..utils.copy import download_files +from ..utils.diff import diff_ascii +from .nxclass_app import diff_nxclass_docs +from .nxclass_app import save_nxclass_docs + + +def manual_args(parser): + parser.add_argument( + "--test", + action="store_true", + help="Validate all NeXus class definitions", + ) + parser.add_argument( + "--prepare", + action="store_true", + help="Create the build files for the NeXus Impatient Guide", + ) + parser.add_argument( + "--diff", + action="store_true", + help="Print all changes in the generated documentation", + ) + + +def manual_exec(args): + # Copy the documentation source files to the build directory + if args.prepare: + copydir(directories.manual_source_root(), directories.manual_build_root()) + + # XSD and NXDL document generators + generate_docs = args.prepare or args.diff + if generate_docs: + generator = NXClassDocGenerator() + xsdgenerator = XSDDocGenerator() + if args.prepare: + output_path = directories.manual_build_staticroot() + else: + output_path = None + anchor_registry = AnchorRegistry(output_path=output_path) + + # Generate the NeXus class documentation files in the build directory + for subdir in ("base_classes", "applications", "contributed_definitions"): + for nxdl_file in iter_definitions(subdir): + if args.test: + validate_definition(nxdl_file) + if generate_docs: + rst_lines = generator(nxdl_file, anchor_registry=anchor_registry) + if args.diff: + diff_nxclass_docs(nxdl_file, rst_lines) + if args.prepare: + save_nxclass_docs(nxdl_file, rst_lines) + + # Generate the NXDL XSD documentation in the build directory + if generate_docs: + xsd_file = directories.get_xsd_file() + rst_lines = xsdgenerator(xsd_file) + nxdl_desc = directories.manual_build_sphinxroot() / "nxdl_desc.rst" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + if generate_docs: + xsd_file = directories.get_xsd_units_file() + rst_lines = generate_xsd_units_doc(xsd_file, "anyUnitsAttr", "units") + nxdl_desc = directories.manual_build_sphinxroot() / "units.table" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation (units)", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + if generate_docs: + xsd_file = directories.get_xsd_units_file() + rst_lines = generate_xsd_units_doc(xsd_file, "primitiveType", "data") + nxdl_desc = directories.manual_build_sphinxroot() / "types.table" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation (types)", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + # Generate the NeXus class documentation index files in the + # build directory so the files generated above are included in the docs. + if generate_docs: + for name, adict in nxdl_indices().items(): + index_file = adict["index_file"] + rst_lines = adict["rst_lines"] + if args.diff: + diff_ascii(name, rst_lines, index_file) + if args.prepare: + print("generate NXDL index", index_file) + if args.prepare: + with open(index_file, "w") as fh: + fh.writelines(rst_lines) + + # Generate the anchor list in several format + if args.prepare: + print("generate anchor list files in", output_path) + anchor_registry.write() + copy_files(EXTRA_FILES) + download_files(EXTRA_URLS) + + +# Path relative to source directory, +# Path relative to build directory, +# Overwrite (boolean) +EXTRA_FILES = [["NXDL_VERSION", "NXDL_VERSION", True], ["LGPL.txt", "LGPL.txt", True]] + +# URL, +# Path relative to build directory, +# Overwrite (boolean) +EXTRA_URLS = [ + [ + "https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf", + "manual/source/_static/NeXusIntern.pdf", + False, + ], + [ + "https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf", + "manual/source/_static/NXtranslate.pdf", + False, + ], +] diff --git a/dev_tools/apps/nxclass_app.py b/dev_tools/apps/nxclass_app.py new file mode 100644 index 0000000000..e0025cfd59 --- /dev/null +++ b/dev_tools/apps/nxclass_app.py @@ -0,0 +1,103 @@ +import shutil +from pathlib import Path +from typing import List + +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..globals import directories +from ..nxdl import find_definition +from ..nxdl import validate_definition +from ..utils.diff import diff_ascii + + +def nxclass_args(parser): + parser.add_argument( + "name", + type=str, + help="NeXus class name (For example 'nxdata')", + ) + parser.add_argument( + "--test", + action="store_true", + help="Validate the NeXus class definition", + ) + parser.add_argument( + "--prepare", + action="store_true", + help="Save the NeXus class documentation in the build directory", + ) + parser.add_argument( + "--diff", + action="store_true", + help="Print changes in the NeXus class documentation", + ) + parser.add_argument( + "--print", + action="store_true", + help="Print the NeXus class documentation", + ) + + +def nxclass_exec(args): + nxdl_file = find_definition(args.name) + assert nxdl_file, f"No definition found for {args.name}" + + if args.test: + validate_definition(nxdl_file) + + if args.prepare or args.diff or args.print: + generator = NXClassDocGenerator() + if args.prepare: + output_path = directories.manual_build_staticroot() + else: + output_path = None + anchor_registry = AnchorRegistry(output_path=output_path) + rst_lines = generator(nxdl_file, anchor_registry=anchor_registry) + + if args.prepare: + save_nxclass_docs(nxdl_file, rst_lines) + print("add to anchor list files in", output_path) + anchor_registry.write() + + if args.diff: + diff_nxclass_docs(nxdl_file, rst_lines) + + if args.print: + print("".join(rst_lines)) + + +def save_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: + """Build the NXDL file: this means prepare the documentation + and resources in the build directory. + """ + rst_file_name = directories.get_rst_filename(nxdl_file) + + # Save the documentation in the build directory + print("generate", nxdl_file) + print(" ->", rst_file_name) + rst_file_name.parent.mkdir(parents=True, exist_ok=True) + with open(rst_file_name, "w") as fh: + fh.writelines(rst_lines) + + # Copy resources to the build directory + resource_name = rst_file_name.stem[2:] # (e.g. NXaperture -> aperture) + nxdl_resource_dir = nxdl_file.parent / resource_name + if nxdl_resource_dir.is_dir(): + rst_resource_dir = rst_file_name.parent / resource_name + print("copy", nxdl_resource_dir) + print(" ->", rst_resource_dir) + shutil.copytree(nxdl_resource_dir, rst_resource_dir, dirs_exist_ok=True) + + # Copy the NXDL file as it might be used in a "literalinclude" + nxdl_file2 = rst_file_name.parent / nxdl_file.name + print("generate", nxdl_file) + print(" ->", nxdl_file2) + shutil.copyfile(nxdl_file, nxdl_file2) + + +def diff_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: + """Build the NXDL file: this means prepare the documentation + and resources in the build directory. + """ + rst_file_name = directories.get_rst_filename(nxdl_file) + diff_ascii(nxdl_file, rst_lines, rst_file_name) diff --git a/dev_tools/docs/__init__.py b/dev_tools/docs/__init__.py new file mode 100644 index 0000000000..a0fcda82c7 --- /dev/null +++ b/dev_tools/docs/__init__.py @@ -0,0 +1,4 @@ +from .anchor_list import AnchorRegistry # noqa F401 +from .nxdl import NXClassDocGenerator # noqa F401 +from .nxdl_index import nxdl_indices # noqa F401 +from .xsd import XSDDocGenerator # noqa F401 diff --git a/dev_tools/docs/anchor_list.py b/dev_tools/docs/anchor_list.py new file mode 100644 index 0000000000..df668ead01 --- /dev/null +++ b/dev_tools/docs/anchor_list.py @@ -0,0 +1,226 @@ +import datetime +import json +import os +from pathlib import Path +from typing import Optional + +import lxml +import yaml + +from ..globals import directories +from ..globals.nxdl import get_nxdl_version +from ..globals.urls import MANUAL_URL +from ..utils.types import PathLike + + +class AnchorRegistry: + """Document the NXDL vocabulary. Usage goes as follows + + .. code:: python + + reg = AnchorRegistry(output_path=...) + # It loaded the existing registry from file (if provided) + + reg.add(...) + reg.add(...) + ... + anchors = reg.flush_anchor_buffer() + + reg.add(...) + reg.add(...) + ... + anchors = reg.flush_anchor_buffer() + + ... + reg.write() + # It saved the in-memory registry to file (if provided) + """ + + def __init__(self, output_path: Optional[PathLike] = None) -> None: + # For example: output_path = get_build_root() / "manual" / "source" / "_static" + self._writing_enabled = bool(output_path) + if output_path: + base = "nxdl_vocabulary" + output_path = Path(output_path).absolute() + output_path.mkdir(parents=True, exist_ok=True) + self._html_file = output_path / f"{base}.html" + self._txt_file = output_path / f"{base}.txt" + self._json_file = output_path / f"{base}.json" + self._yaml_file = output_path / f"{base}.yml" + else: + self._html_file = None + self._txt_file = None + self._json_file = None + self._yaml_file = None + self._registry = self._load_registry() + self._anchor_buffer = [] + self._nxdl_file = None + + @property + def all_anchors(self): + result = [] + for v in self._registry.values(): + result += list(v.keys()) + return result + + @property + def nxdl_file(self) -> Optional[Path]: + return self._nxdl_file + + @nxdl_file.setter + def nxdl_file(self, value: PathLike) -> None: + self._nxdl_file = Path(value).absolute() + + @property + def html_url(self): + rst_file = directories.get_rst_filename(self.nxdl_file) + manual_root = directories.manual_build_sphinxroot() + rel_path = rst_file.relative_to(manual_root) + rel_html = str(rel_path.with_suffix(".html")) + rel_html = rel_html.replace(os.sep, "/") + return f"{MANUAL_URL}/{rel_html}" + + def add(self, anchor): + """Add anchor to the in-memory registry and to the + current anchor buffer.""" + if anchor not in self._anchor_buffer: + self._anchor_buffer.append(anchor) + + key = self._key_from_anchor(anchor) + + if key not in self._registry: + self._registry[key] = {} + + reg = self._registry[key] + if anchor not in reg: + hanchor = self._html_anchor(anchor) + url = f"{self.html_url}{hanchor}" + reg[anchor] = dict( + term=anchor, + html=hanchor, + url=url, + ) + + def _key_from_anchor(self, anchor): + key = anchor.lower().split("/")[-1].split("@")[-1].split("-")[0] + if "@" in anchor: + # restore preceding "@" symbol + key = "@" + key + return key + + def write(self): + """Write the in-memory registry to files""" + if not self._writing_enabled: + return + contents = dict( + _metadata=dict( + datetime=datetime.datetime.utcnow().isoformat(), + title="NeXus NXDL vocabulary.", + subtitle="Anchors for all NeXus fields, groups, " + "attributes, and links.", + version=get_nxdl_version(), + ), + terms=self._registry, + ) + self._write_yaml(contents) + self._write_json(contents) + self._write_txt() + self._write_html(contents) + + def flush_anchor_buffer(self) -> list: + """Flush the anchor buffer""" + self._anchor_buffer, ret = list(), self._anchor_buffer + return ret + + def _html_anchor(self, anchor): + """ + Create (internal hyperlink target for) HTML anchor from reST anchor. + + Example: + + * reST anchor: /NXcanSAS/ENTRY/TRANSMISSION_SPECTRUM@timestamp-attribute + * HTML anchor: #nxcansas-entry-transmission-spectrum-timestamp-attribute + """ + html_anchor = ( + anchor.lower() + .lstrip("/") + .replace("_", "-") + .replace("@", "-") + .replace("/", "-") + ) + return f"#{html_anchor}" + + def _load_registry(self) -> dict: + """Load the anchor registry in memory.""" + if not self._yaml_file: + return {} + registry = None + if self._yaml_file.exists(): + contents = yaml.load(open(self._yaml_file, "r").read(), Loader=yaml.Loader) + if contents is not None: + registry = contents.get("terms") + return registry or {} + + def _write_html(self, contents): + """Write the anchors to an HTML file.""" + if not self._html_file: + return + root = lxml.etree.Element("html") + body = lxml.etree.SubElement(root, "body") + title = lxml.etree.SubElement(body, "h1") + subtitle = lxml.etree.SubElement(body, "em") + + title.text = contents["_metadata"]["title"].strip(".") + subtitle.text = contents["_metadata"]["subtitle"].strip(".") + vocab_list = lxml.etree.SubElement(body, "h2") + vocab_list.text = "NXDL Vocabulary" + + p = lxml.etree.SubElement(body, "p") + p.text = "This content is also available in these formats: " + for ext in "json txt yml".split(): + a = lxml.etree.SubElement(p, "a") + a.attrib["href"] = f"{MANUAL_URL}/_static/{self._txt_file.stem}.{ext}" + a.text = f" {ext}" + + dl = lxml.etree.SubElement(body, "dl") + for term, termlist in sorted(contents["terms"].items()): + dterm = lxml.etree.SubElement(dl, "dt") + dterm.text = term + for _, itemdict in sorted(termlist.items()): + ddef = lxml.etree.SubElement(dterm, "dd") + a = lxml.etree.SubElement(ddef, "a") + a.attrib["href"] = itemdict["url"] + a.text = itemdict["term"] + + lxml.etree.SubElement(body, "hr") + + foot = lxml.etree.SubElement(body, "p") + foot_em = lxml.etree.SubElement(foot, "em") + foot_em.text = f"written: {contents['_metadata']['datetime']}" + + html = lxml.etree.tostring(root, pretty_print=True).decode() + with open(self._html_file, "w") as f: + f.write(html) + f.write("\n") + + def _write_json(self, contents): + if not self._json_file: + return + with open(self._json_file, "w") as f: + json.dump(contents, f, indent=4, sort_keys=True) + f.write("\n") + + def _write_txt(self): + """Compendium (dump the list of all known anchors in raw form).""" + if not self._txt_file: + return + terms = self.all_anchors + with open(self._txt_file, "w") as f: + f.write("\n".join(sorted(terms))) + f.write("\n") + + def _write_yaml(self, contents): + if not self._yaml_file: + return + with open(self._yaml_file, "w") as f: + yaml.dump(contents, f) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py new file mode 100644 index 0000000000..a7a355cc2d --- /dev/null +++ b/dev_tools/docs/nxdl.py @@ -0,0 +1,631 @@ +import os +import re +from collections import OrderedDict +from html import parser as HTMLParser +from pathlib import Path +from typing import List +from typing import Optional + +import lxml + +from ..globals.directories import get_nxdl_root +from ..globals.errors import NXDLParseError +from ..globals.nxdl import NXDL_NAMESPACE +from ..globals.urls import REPO_URL +from ..utils.types import PathLike +from .anchor_list import AnchorRegistry + + +class NXClassDocGenerator: + """Generate documentation in reStructuredText markup + for a NeXus class definition.""" + + _INDENTATION_UNIT = " " * 2 + + _CATEGORY_TO_LISTING = { + "base": "base class", + "application": "application definition", + } + + def __init__(self) -> None: + self._rst_lines = None + self._reset() + + def _reset(self): + self._anchor_registry = None + self._listing_category = None + self._use_application_defaults = None + + def __call__( + self, nxdl_file: PathLike, anchor_registry: Optional[AnchorRegistry] = None + ) -> List[str]: + self._rst_lines = list() + self._anchor_registry = anchor_registry + nxdl_file = Path(nxdl_file) + if anchor_registry: + self._anchor_registry.nxdl_file = nxdl_file + try: + try: + self._parse_nxdl_file(nxdl_file) + except Exception: + raise NXDLParseError(nxdl_file) + finally: + self._reset() + return self._rst_lines + + def _parse_nxdl_file(self, nxdl_file: Path): + assert nxdl_file.is_file() + tree = lxml.etree.parse(str(nxdl_file)) + root = tree.getroot() + + # NXDL_NAMESPACE needs to be a globally unique identifier of + # the NXDL schema. It needs to match the xmlns attribute + # in the NXDL definition of the NeXus class. + ns = {"nx": NXDL_NAMESPACE} + + nxclass_name = root.get("name") + category = root.attrib["category"] + title = nxclass_name + parent_path = "/" + nxclass_name # absolute path of parent nodes, no trailing / + if len(nxclass_name) < 2 or nxclass_name[0:2] != "NX": + raise Exception( + f'Unexpected class name "{nxclass_name}"; does not start with NX' + ) + lexical_name = nxclass_name[2:] # without padding 'NX', for indexing + self._listing_category = self._CATEGORY_TO_LISTING[category] + self._use_application_defaults = category == "application" + + # print ReST comments and section header + source = os.path.relpath(nxdl_file, get_nxdl_root()) + self._print( + f".. auto-generated by {__name__} from the NXDL source {source} -- DO NOT EDIT" + ) + self._print("") + self._print(".. index::") + self._print(f" ! {nxclass_name} ({self._listing_category})") + self._print(f" ! {lexical_name} ({self._listing_category})") + self._print( + f" see: {lexical_name} ({self._listing_category}); {nxclass_name}" + ) + self._print("") + self._print(f".. _{nxclass_name}:\n") + self._print("=" * len(title)) + self._print(title) + self._print("=" * len(title)) + + # print category & parent class + extends = root.get("extends") + if extends is None: + extends = "none" + else: + extends = f":ref:`{extends}`" + + self._print("") + self._print("**Status**:\n") + self._print(f" {self._listing_category.strip()}, extends {extends}") + + self._print_if_deprecated(ns, root, "") + + # print official description of this class + self._print("") + self._print("**Description**:\n") + self._print_doc(self._INDENTATION_UNIT, ns, root, required=True) + + # print symbol list + node_list = root.xpath("nx:symbols", namespaces=ns) + self._print("**Symbols**:\n") + if len(node_list) == 0: + self._print(" No symbol table\n") + elif len(node_list) > 1: + raise Exception(f"Invalid symbol table in {nxclass_name}") + else: + self._print_doc(self._INDENTATION_UNIT, ns, node_list[0]) + for node in node_list[0].xpath("nx:symbol", namespaces=ns): + doc = self._get_doc_line(ns, node) + self._print(f" **{node.get('name')}**", end="") + if doc: + self._print(f": {doc}", end="") + self._print("\n") + + # print group references + self._print("**Groups cited**:") + node_list = root.xpath("//nx:group", namespaces=ns) + groups = [] + for node in node_list: + g = node.get("type") + if g.startswith("NX") and g not in groups: + groups.append(g) + if len(groups) == 0: + self._print(" none\n") + else: + out = [(f":ref:`{g}`") for g in groups] + txt = ", ".join(sorted(out)) + self._print(f" {txt}\n") + out = [ + ("%s (base class); used in %s" % (g, self._listing_category)) + for g in groups + ] + txt = ", ".join(out) + self._print(f".. index:: {txt}\n") + + # print full tree + self._print("**Structure**:\n") + for subnode in root.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, "file", subnode, optional, self._INDENTATION_UNIT, parent_path + ) # FIXME: +"/"+name ) + self._print_full_tree( + ns, root, nxclass_name, self._INDENTATION_UNIT, parent_path + ) + + self._print_anchor_list() + + # print NXDL source location + self._print("") + self._print("**NXDL Source**:") + nxdl_root = get_nxdl_root() + rel_path = str(nxdl_file.relative_to(nxdl_root)) + rel_html = str(rel_path).replace(os.sep, "/") + self._print(f" {REPO_URL}/{rel_html}") + + return self._rst_lines + + def _print_anchor_list(self): + """Print the list of hypertext anchors.""" + if not self._anchor_registry: + return + anchors = self._anchor_registry.flush_anchor_buffer() + if not anchors: + return + + self._print("") + self._print("Hypertext Anchors") + self._print("-----------------\n") + self._print( + "List of hypertext anchors for all groups, fields,\n" + "attributes, and links defined in this class.\n\n" + ) + + def sorter(key): + return key.lower() + + rst = [f"* :ref:`{ref} <{ref}>`" for ref in sorted(anchors, key=sorter)] + + self._print("\n".join(rst)) + + @staticmethod + def _format_type(node): + typ = node.get("type", ":ref:`NX_CHAR `") # per default + if typ.startswith("NX_"): + typ = f":ref:`{typ} <{typ}>`" + return typ + + @staticmethod + def _format_units(node): + units = node.get("units", "") + if not units: + return "" + if units.startswith("NX_"): + units = rf"\ :ref:`{units} <{units}>`" + return f" {{units={units}}}" + + @staticmethod + def _get_doc_blocks(ns, node): + docnodes = node.xpath("nx:doc", namespaces=ns) + if docnodes is None or len(docnodes) == 0: + return "" + if len(docnodes) > 1: + raise Exception( + f"Too many doc elements: line {node.sourceline}, {Path(node.base).name}" + ) + docnode = docnodes[0] + + # be sure to grab _all_ content in the documentation + # it might look like XML + s = lxml.etree.tostring( + docnode, pretty_print=True, method="c14n", with_comments=False + ).decode("utf-8") + m = re.search(r"^]*>\n?(.*)\n?$", s, re.DOTALL) + if not m: + raise Exception(f"unexpected docstring [{s}] ") + text = m.group(1) + + # substitute HTML entities in markup: "<" for "<" + # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string + htmlparser = HTMLParser.HTMLParser() + try: # see #661 + import html + + text = html.unescape(text) + except (ImportError, AttributeError): + text = htmlparser.unescape(text) + + # Blocks are separated by whitelines + blocks = re.split("\n\\s*\n", text) + if len(blocks) == 1 and len(blocks[0].splitlines()) == 1: + return [blocks[0].rstrip().lstrip()] + + # Indentation must be given by first line + m = re.match(r"(\s*)(\S+)", blocks[0]) + if not m: + return [""] + indent = m.group(1) + + # Remove common indentation as determined from first line + if indent == "": + raise Exception( + "Missing initial indentation in of %s [%s]" + % (node.get("name"), blocks[0]) + ) + + out_blocks = [] + for block in blocks: + lines = block.rstrip().splitlines() + out_lines = [] + for line in lines: + if line[: len(indent)] != indent: + raise Exception( + 'Bad indentation in of %s [%s]: expected "%s" found "%s".' + % ( + node.get("name"), + block, + re.sub(r"\t", "\\\\t", indent), + re.sub(r"\t", "\\\\t", line), + ) + ) + out_lines.append(line[len(indent) :]) + out_blocks.append("\n".join(out_lines)) + return out_blocks + + def _get_doc_line(self, ns, node): + blocks = self._get_doc_blocks(ns, node) + if len(blocks) == 0: + return "" + if len(blocks) > 1: + raise Exception(f"Unexpected multi-paragraph doc [{'|'.join(blocks)}]") + return re.sub(r"\n", " ", blocks[0]) + + def _get_minOccurs(self, node): + """ + get the value for the ``minOccurs`` attribute + + :param obj node: instance of lxml.etree._Element + :returns str: value of the attribute (or its default) + """ + # TODO: can we improve on the default by examining nxdl.xsd? + minOccurs_default = str(int(self._use_application_defaults)) + minOccurs = node.get("minOccurs", minOccurs_default) + return minOccurs + + def _get_required_or_optional_text(self, node): + """ + make clear if a reported item is required or optional + + :param obj node: instance of lxml.etree._Element + :returns: formatted text + """ + tag = node.tag.split("}")[-1] + if tag in ("field", "group"): + optional_default = not self._use_application_defaults + optional = node.get("optional", optional_default) in (True, "true", "1", 1) + recommended = node.get("recommended", None) in (True, "true", "1", 1) + minOccurs = self._get_minOccurs(node) + if recommended: + optional_text = "(recommended) " + elif minOccurs in ("0", 0) or optional: + optional_text = "(optional) " + elif minOccurs in ("1", 1): + optional_text = "(required) " + else: + # this is unexpected and remarkable + # TODO: add a remark to the log + optional_text = f"(``minOccurs={str(minOccurs)}``) " + elif tag in ("attribute",): + optional_default = not self._use_application_defaults + optional = node.get("optional", optional_default) in (True, "true", "1", 1) + recommended = node.get("recommended", None) in (True, "true", "1", 1) + optional_text = {True: "(optional) ", False: "(required) "}[optional] + if recommended: + optional_text = "(recommended) " + else: + optional_text = "(unknown tag: " + str(tag) + ") " + return optional_text + + def _analyze_dimensions(self, ns, parent) -> str: + """These are the different dimensions that can occur: + + 1. Fixed rank + + + + + + + + 2. Variable rank because of optional dimensions + + + + + + + + + 3. Variable rank because no dimensions specified + + + + + The legacy way of doing this (still supported) + + + + + + 4. Rank and dimensions equal to that of another field called `field_name` + + + + + """ + node_list = parent.xpath("nx:dimensions", namespaces=ns) + if len(node_list) != 1: + return "" + node = node_list[0] + node_list = node.xpath("nx:dim", namespaces=ns) + + dims = [] + optional = False + for subnode in node_list: + # Dimension index (starts from index 1) + index = subnode.get("index", "") + if not index.isdigit(): + raise RuntimeError("A dimension must have an index") + index = int(index) + if index == 0: + # No longer needed: legacy way to specify that the + # rank is variable + continue + + # Expand dimensions when needed + index -= 1 + nadd = max(index - len(dims) + 1, 0) + if nadd: + dims += ["."] * nadd + + # Dimension symbol + dim = subnode.get("value") # integer or symbol from the table + if not dim: + ref = subnode.get("ref") + if ref: + return ( + f" (Rank: same as field {ref}, Dimensions: same as field {ref})" + ) + dim = "." # dimension has no symbol + + # Dimension might be optional + if subnode.get("required", "true").lower() == "false": + optional = True + elif optional: + raise RuntimeError( + "A required dimension cannot come after an optional dimension" + ) + if optional: + dim = f"[{dim}]" + + dims[index] = dim + + # When the rank is missing, set to the number of dimensions when + # there are dimensions specified and none of them are optional. + ndims = len(dims) + rank = node.get("rank", None) + if rank is None and not optional and ndims: + rank = str(ndims) + + # Validate rank and dimensions + rank_is_fixed = rank and rank.isdigit() + if optional and rank_is_fixed: + raise RuntimeError("A fixed rank cannot have optional dimensions") + if rank_is_fixed and ndims and int(rank) != ndims: + raise RuntimeError( + "The rank and the number of dimensions do not correspond" + ) + + # Omit rank and/or dimensions when not specified + if rank and dims: + dims = ", ".join(dims) + return f" (Rank: {rank}, Dimensions: [{dims}])" + elif rank: + return f" (Rank: {rank})" + elif dims: + dims = ", ".join(dims) + return f" (Dimensions: [{dims}])" + return "" + + def _hyperlink_target(self, parent_path, name, nxtype): + """Return internal hyperlink target for HTML anchor.""" + if nxtype == "attribute": + sep = "@" + else: + sep = "/" + target = f"{parent_path}{sep}{name}-{nxtype}" + if self._anchor_registry: + self._anchor_registry.add(target) + return f".. _{target}:\n" + + def _print_enumeration(self, indent, ns, parent): + node_list = parent.xpath("nx:item", namespaces=ns) + if len(node_list) == 0: + return "" + + if len(node_list) == 1: + self._print(f"{indent}Obligatory value:", end="") + else: + self._print(f"{indent}Any of these values:", end="") + + docs = OrderedDict() + for item in node_list: + name = item.get("value") + docs[name] = self._get_doc_line(ns, item) + + ENUMERATION_INLINE_LENGTH = 60 + + def show_as_typed_text(msg): + return f"``{msg}``" + + oneliner = " | ".join(map(show_as_typed_text, docs.keys())) + if ( + any(doc for doc in docs.values()) + or len(oneliner) > ENUMERATION_INLINE_LENGTH + ): + # print one item per line + self._print("\n") + for name, doc in docs.items(): + self._print(f"{indent} * {show_as_typed_text(name)}", end="") + if doc: + self._print(f": {doc}", end="") + self._print("\n") + else: + # print all items in one line + self._print(f" {oneliner}") + self._print("") + + def _print_doc(self, indent, ns, node, required=False): + blocks = self._get_doc_blocks(ns, node) + if len(blocks) == 0: + if required: + raise Exception("No documentation for: " + node.get("name")) + self._print("") + else: + for block in blocks: + for line in block.splitlines(): + self._print(f"{indent}{line}") + self._print() + + def _print_attribute(self, ns, kind, node, optional, indent, parent_path): + name = node.get("name") + index_name = name + self._print( + f"{indent}" f"{self._hyperlink_target(parent_path, name, 'attribute')}" + ) + self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") + self._print( + f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n" + ) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + node_list = node.xpath("nx:enumeration", namespaces=ns) + if len(node_list) == 1: + self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0]) + + def _print_if_deprecated(self, ns, node, indent): + deprecated = node.get("deprecated", None) + if deprecated is not None: + self._print(f"\n{indent}.. index:: deprecated\n") + self._print(f"\n{indent}**DEPRECATED**: {deprecated}\n") + + def _print_full_tree(self, ns, parent, name, indent, parent_path): + """ + recursively print the full tree structure + + :param dict ns: dictionary of namespaces for use in XPath expressions + :param lxml_element_node parent: parent node to be documented + :param str name: name of elements, such as NXentry/NXuser + :param indent: to keep track of indentation level + :param parent_path: NX class path of parent nodes + """ + + self._use_application_defaults = self._listing_category in ( + "application definition", + "contributed definition", + ) + + for node in parent.xpath("nx:field", namespaces=ns): + name = node.get("name") + index_name = name + dims = self._analyze_dimensions(ns, node) + + optional_text = self._get_required_or_optional_text(node) + self._print(f"{indent}{self._hyperlink_target(parent_path, name, 'field')}") + self._print(f"{indent}.. index:: {index_name} (field)\n") + self._print( + f"{indent}**{name}**: " + f"{optional_text}" + f"{self._format_type(node)}" + f"{dims}" + f"{self._format_units(node)}" + "\n" + ) + + self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + node_list = node.xpath("nx:enumeration", namespaces=ns) + if len(node_list) == 1: + self._print_enumeration( + indent + self._INDENTATION_UNIT, ns, node_list[0] + ) + + for subnode in node.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, + "field", + subnode, + optional, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + for node in parent.xpath("nx:group", namespaces=ns): + name = node.get("name", "") + typ = node.get("type", "untyped (this is an error; please report)") + + optional_text = self._get_required_or_optional_text(node) + if typ.startswith("NX"): + if name == "": + name = typ.lstrip("NX").upper() + typ = f":ref:`{typ}`" + hTarget = self._hyperlink_target(parent_path, name, "group") + # target = hTarget.replace(".. _", "").replace(":\n", "") + # TODO: https://github.com/nexusformat/definitions/issues/1057 + self._print(f"{indent}{hTarget}") + self._print(f"{indent}**{name}**: {optional_text}{typ}\n") + + self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + for subnode in node.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, + "group", + subnode, + optional, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + nodename = "%s/%s" % (name, node.get("type")) + self._print_full_tree( + ns, + node, + nodename, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + for node in parent.xpath("nx:link", namespaces=ns): + name = node.get("name") + self._print(f"{indent}{self._hyperlink_target(parent_path, name, 'link')}") + self._print( + f"{indent}**{name}**: " + ":ref:`link` " + f"(suggested target: ``{node.get('target')}``" + "\n" + ) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + def _print(self, *args, end="\n"): + # TODO: change instances of \t to proper indentation + self._rst_lines.append(" ".join(args) + end) diff --git a/utils/nxdl_summary.py b/dev_tools/docs/nxdl_index.py old mode 100755 new mode 100644 similarity index 51% rename from utils/nxdl_summary.py rename to dev_tools/docs/nxdl_index.py index a8cbc67162..98f90b95bc --- a/utils/nxdl_summary.py +++ b/dev_tools/docs/nxdl_index.py @@ -1,25 +1,83 @@ -#!/usr/bin/env python +import os +from pathlib import Path +from typing import Dict -""" -Summarize the NXDL classes definitions for the given NXDL section. - -Re-write the index.rst file with a list of: class summary (and a hidden toctree) -""" - - -import os, sys import lxml.etree +from ..globals import directories +from ..globals.nxdl import NXDL_NAMESPACE +from ..nxdl import iter_definitions -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 +def nxdl_indices() -> Dict[str, dict]: + """For each directory under the NXDL root, create the content of an + index file which link all NeXus class definition doc files. + """ + indentation = " " * 4 + namespaces = {"nx": NXDL_NAMESPACE} + sections = dict() + root = directories.get_nxdl_root() + + for nxdl_file in iter_definitions(): + source = os.path.relpath(nxdl_file.parent, root) + section = sections.get(source) + if section is None: + preample = _DIRNAME_TO_PREAMBLE.get(source) + if preample is None: + continue + classes = [] + rst_lines = [] + index_file = directories.nxclass_build_root(nxdl_file) / "index.rst" + sections[source] = { + "index_file": index_file, + "rst_lines": rst_lines, + "classes": classes, + } + rst_lines.append(".. do NOT edit this file\n") + rst_lines.append(f" automatically generated by {__name__}\n") + rst_lines.append("\n") + rst_lines.append(preample) + rst_lines.append("\n") + else: + classes = sections[source]["classes"] + rst_lines = sections[source]["rst_lines"] + + nxclass_name = nxdl_file.with_suffix("").stem + classes.append(nxclass_name) + summary = get_nxclass_description(nxdl_file, namespaces) + rst_lines.append("\n") + rst_lines.append(f":ref:`{nxclass_name}`\n") + rst_lines.append(f"{indentation}{summary}\n") + + # Create a table of content for each index file + for section in sections.values(): + classes = section.pop("classes") + rst_lines = section["rst_lines"] + rst_lines.append("\n") + rst_lines.append(".. toctree::\n") + rst_lines.append(f"{indentation}:hidden:\n") + rst_lines.append("\n") + for cname in sorted(classes): + rst_lines.append(f"{indentation}{cname}\n") + + return sections + + +def get_nxclass_description(nxdl_file: Path, namespaces) -> str: + """ + get the summary line from each NXDL definition doc -NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" -NS = {"nx": NAMESPACE} + That's the first physical line of the doc string. + """ + tree = lxml.etree.parse(nxdl_file) + root = tree.getroot() + nodes = root.xpath("nx:doc", namespaces=namespaces) + if len(nodes) != 1: + raise RuntimeError(f"wrong number of nodes in NXDL: {nxdl_file}") + return nodes[0].text.strip().splitlines()[0] -PREAMBLES = { +_DIRNAME_TO_PREAMBLE = { "base_classes": """ .. index:: ! see: class definitions; base class @@ -55,7 +113,7 @@ are optional in the NeXus data file. The definition, in this case, reserves the exact term by declaring its spelling and description. Consider an application definition as a *contract* -between a data provider (such as the beam line control system) and a +between a data provider (such as the beam line control system) and a data consumer (such as a data analysis program for a scientific technique) that describes the information is certain to be available in a data file. @@ -80,84 +138,8 @@ the community for NeXus base classes or application definitions, as well as other NXDL files for long-term archival by NeXus. Consider the contributed definitions as either in *incubation* or a special -case not for general use. The :ref:`NIAC` is charged to review any new contributed +case not for general use. The :ref:`NIAC` is charged to review any new contributed definitions and provide feedback to the authors before ratification and acceptance as either a base class or application definition. """, } - - -def getSummary(nxdl_file): - """ - get the summary line from each NXDL definition doc - - That's the first physical line of the doc string. - """ - tree = lxml.etree.parse(nxdl_file) - root = tree.getroot() - nodes = root.xpath("nx:doc", namespaces=NS) - if len(nodes) != 1: - raise RuntimeError("wrong number of nodes in NXDL: " + nxdl_file) - text = nodes[0].text - return text.strip().splitlines()[0] - - -def command_args(): - """get the command-line arguments, handle syntax errors""" - import argparse - - doc = __doc__.strip().splitlines()[0] - parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "section", action="store", help="NXDL section (such as *base_classes*)" - ) - return parser.parse_args() - - -def main(section): - if section not in PREAMBLES.keys(): - raise KeyError("unknown NXDL section: " + section) - base_path = os.path.abspath(os.path.dirname(__file__)) - nxdl_path = os.path.abspath(os.path.join(base_path, "..", section)) - if not os.path.exists(nxdl_path): - raise IOError("not found: " + nxdl_path) - - rst_path = os.path.abspath( - os.path.join(base_path, "..", "manual", "source", "classes", section) - ) - if not os.path.exists(rst_path): - raise IOError("not found: " + rst_path) - - index_file = os.path.join(rst_path, "index.rst") - - classes = [] - text = [] - text.append( - """ -.. do NOT edit this file - automatically generated by script """ - + __file__ - ) - text.append("") - text.append(PREAMBLES[section]) - for fname in sorted(os.listdir(nxdl_path)): - if fname.endswith(".nxdl.xml"): - class_name = fname.split(".")[0] - classes.append(class_name) - summary = getSummary(os.path.join(nxdl_path, fname)) - text.append("") - text.append(":ref:`" + class_name + "`") - text.append(INDENTATION + summary) - text.append("") - text.append(".. toctree::") - text.append(INDENTATION + ":hidden:") - text.append("") - for cname in sorted(classes): - text.append(INDENTATION + cname) - text.append("") - open(index_file, "w").writelines("\n".join(text)) - - -if __name__ == "__main__": - cli = command_args() - main(cli.section) diff --git a/dev_tools/docs/xsd.py b/dev_tools/docs/xsd.py new file mode 100644 index 0000000000..88c2e0fe98 --- /dev/null +++ b/dev_tools/docs/xsd.py @@ -0,0 +1,503 @@ +import textwrap +from pathlib import Path +from typing import List + +import lxml.etree + +from ..globals import directories +from ..globals.errors import NXDLParseError +from ..globals.nxdl import XSD_NAMESPACE +from ..utils.types import PathLike + + +class XSDDocGenerator: + """ + Read the NXDL field types specification and find + all the valid data types. Write a restructured + text (.rst) document for use in the NeXus manual in + the NXDL chapter. + """ + + TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles + INDENTATION = " " * 4 + DATATYPE_DICT = { + "basicComponent": """/xs:schema//xs:complexType[@name='basicComponent']""", + "validItemName": """/xs:schema//xs:simpleType[@name='validItemName']""", + "validNXClassName": """/xs:schema//xs:simpleType[@name='validNXClassName']""", + "validTargetName": """/xs:schema//xs:simpleType[@name='validTargetName']""", + "nonNegativeUnbounded": """/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']""", + } + + def __init__(self) -> None: + self.ns = {"xs": XSD_NAMESPACE} + self._rst_lines = None + + def __call__(self, xsd_file: PathLike) -> List[str]: + self._rst_lines = list() + xsd_file = Path(xsd_file) + try: + self._parse_xsd_file(xsd_file) + except Exception: + raise NXDLParseError(xsd_file) + return self._rst_lines + + def _parse_xsd_file(self, xsd_file: Path): + tree = lxml.etree.parse(str(xsd_file)) + + self._print(f".. auto-generated by {__name__} -- DO NOT EDIT") + self._print(ELEMENT_PREAMBLE) + + for name in sorted(ELEMENT_DICT): + self._print("") + self._print(".. index:: ! %s (NXDL element)\n" % name) + self._print(".. _%s:\n" % name) + self.print_title(name, indentLevel=0) + self._print("\n") + self._print(ELEMENT_DICT[name]) + self._print("\n") + self.add_figure(name, indentLevel=0) + + self._print(DATATYPE_PREAMBLE) + + path_list = ( + "/xs:schema/xs:complexType[@name='attributeType']", + "/xs:schema/xs:element[@name='definition']", + "/xs:schema/xs:complexType[@name='definitionType']", + "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", + "/xs:schema/xs:complexType[@name='dimensionsType']", + "/xs:schema/xs:complexType[@name='docType']", + "/xs:schema/xs:complexType[@name='enumerationType']", + "/xs:schema/xs:complexType[@name='fieldType']", + "/xs:schema/xs:complexType[@name='choiceType']", + "/xs:schema/xs:complexType[@name='groupType']", + "/xs:schema/xs:complexType[@name='linkType']", + "/xs:schema/xs:complexType[@name='symbolsType']", + "/xs:schema/xs:complexType[@name='basicComponent']", + "/xs:schema/xs:simpleType[@name='validItemName']", + "/xs:schema/xs:simpleType[@name='validNXClassName']", + "/xs:schema/xs:simpleType[@name='validTargetName']", + "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", + ) + for path in path_list: + nodes = self.pick_nodes_from_xpath(tree, path) + self._print("\n.. Xpath = %s\n" % path) + self.general_handler(parent=nodes[0]) + + self._print(DATATYPE_POSTAMBLE) + + def _tag_match(self, parent, match_list): + """match this tag to a list""" + if parent is None: + raise ValueError("Must supply a valid parent node") + parent_tag = parent.tag + tag_found = False + for item in match_list: + # this routine only handles certain XML Schema components + tag_found = parent_tag == "{%s}%s" % (self.ns["xs"], item) + if tag_found: + break + return tag_found + + def _indent(self, indentLevel): + return self.INDENTATION * indentLevel + + def print_title(self, title, indentLevel): + self._print(title) + self._print(self.TITLE_MARKERS[indentLevel] * len(title) + "\n") + + def general_handler(self, parent=None, indentLevel=0): + """Handle XML nodes like the former XSLT template""" + # ignore things we don't know how to handle + known_tags = ("complexType", "simpleType", "group", "element", "attribute") + if not self._tag_match(parent, known_tags): + return + + parent_name = parent.get("name") + if parent_name is None: + return + + simple_tag = parent.tag[ + parent.tag.find("}") + 1 : + ] # cut off the namespace identifier + + # ... + name = parent_name # + ' data type' + if simple_tag == "attribute": + name = "@" + name + + if indentLevel == 0 and simple_tag not in ("attribute"): + self._print(f".. index:: ! {name} (NXDL data type)\n") + self._print(f"\n.. _NXDL.data.type.{name}:\n") + + self.print_title(name, indentLevel) + + self.print_docs(parent, indentLevel) + + if len(parent.xpath("xs:attribute", namespaces=self.ns)) > 0: + self.print_title("Attributes of " + name, indentLevel + 1) + self.apply_templates(parent, "xs:attribute", indentLevel + 1) + + node_list = parent.xpath("xs:restriction", namespaces=self.ns) + if len(node_list) > 0: + # print_title("Restrictions of "+name, indentLevel+1) + self.restriction_handler(node_list[0], indentLevel + 1) + node_list = parent.xpath( + "xs:simpleType/xs:restriction/xs:enumeration", namespaces=self.ns + ) + if len(node_list) > 0: + # print_title("Enumerations of "+name, indentLevel+1) + self.apply_templates( + parent, + "xs:simpleType/xs:restriction", + indentLevel + 1, + handler=self.restriction_handler, + ) + + if len(parent.xpath("xs:sequence/xs:element", namespaces=self.ns)) > 0: + self.print_title("Elements of " + name, indentLevel + 1) + self.apply_templates(parent, "xs:sequence/xs:element", indentLevel + 1) + + node_list = parent.xpath("xs:sequence/xs:group", namespaces=self.ns) + if len(node_list) > 0: + self.print_title("Groups under " + name, indentLevel + 1) + self.print_docs(node_list[0], indentLevel + 1) + + self.apply_templates(parent, "xs:simpleType", indentLevel + 1) + self.apply_templates(parent, "xs:complexType", indentLevel + 1) + self.apply_templates(parent, "xs:complexType/xs:attribute", indentLevel + 1) + self.apply_templates( + parent, "xs:complexContent/xs:extension/xs:attribute", indentLevel + 1 + ) + self.apply_templates( + parent, "xs:complexType/xs:sequence/xs:attribute", indentLevel + 1 + ) + self.apply_templates( + parent, "xs:complexType/xs:sequence/xs:element", indentLevel + 1 + ) + self.apply_templates( + parent, + "xs:complexContent/xs:extension/xs:sequence/xs:element", + indentLevel + 1, + ) + + def restriction_handler(self, parent=None, indentLevel=0): + """Handle XSD restriction nodes like the former XSLT template""" + if not self._tag_match(parent, ("restriction",)): + return + self.print_docs(parent, indentLevel) + self._print("\n") + self._print(self._indent(indentLevel) + "The value may be any") + base = parent.get("base") + pattern_nodes = parent.xpath("xs:pattern", namespaces=self.ns) + enumeration_nodes = parent.xpath("xs:enumeration", namespaces=self.ns) + if len(pattern_nodes) > 0: + self._print( + self._indent(indentLevel) + + "``%s``" % base + + " that *also* matches the regular expression::\n" + ) + self._print( + self._indent(indentLevel) + " " * 4 + pattern_nodes[0].get("value") + ) + elif len(pattern_nodes) > 0: + # how will this be reached? Perhaps a deprecated procedure + self._print( + self._indent(indentLevel) + "``%s``" % base + " from this list:" + ) + for node in enumeration_nodes: + self.enumeration_handler(node, indentLevel) + self.print_docs(node, indentLevel) + self._print(self._indent(indentLevel)) + elif len(enumeration_nodes) > 0: + self._print(self._indent(indentLevel) + "one from this list only:\n") + for node in enumeration_nodes: + self.enumeration_handler(node, indentLevel) + self.print_docs(parent, indentLevel) + self._print(self._indent(indentLevel)) + else: + self._print("@" + base) + self._print("\n") + + def enumeration_handler(self, parent=None, indentLevel=0): + """Handle XSD enumeration nodes like the former XSLT template""" + if not self._tag_match(parent, ["enumeration"]): + return + self._print(self._indent(indentLevel) + "* ``%s``" % parent.get("value")) + self.print_docs(parent, indentLevel) + + def apply_templates(self, parent, path, indentLevel, handler=None): + """iterate the nodes found on the supplied XPath expression""" + if handler is None: + handler = self.general_handler + db = {} + for node in parent.xpath(path, namespaces=self.ns): + name = node.get("name") or node.get("ref") or node.get("value") + if name is not None: + if name in ("nx:groupGroup",): + self._print(">" * 45, name) + if name in db: + raise KeyError("Duplicate name found: " + name) + db[name] = node + for name in sorted(db): + node = db[name] + handler(node, indentLevel) + # self.print_docs(node, indentLevel) + + def print_docs(self, parent, indentLevel=0): + docs = self.get_doc_from_node(parent) + if docs is not None: + self._print(self._indent(indentLevel) + "\n") + for line in docs.splitlines(): + self._print(self._indent(indentLevel) + line) + self._print(self._indent(indentLevel) + "\n") + + def get_doc_from_node(self, node, retval=None): + annotation_node = node.find("xs:annotation", self.ns) + if annotation_node is None: + return retval + documentation_node = annotation_node.find("xs:documentation", self.ns) + if documentation_node is None: + return retval + + # Be sure to grab _all_ content in the node. + # In the documentation nodes, use XML entities ("<"" instead of "<") + # for documentation characters that would otherwise be considered as XML. + s = lxml.etree.tostring(documentation_node, method="text", pretty_print=True) + rst = s.decode().lstrip("\n") # remove any leading blank lines + rst = rst.rstrip() # remove any trailing white space + text = textwrap.dedent(rst) # remove common leading space + + # substitute HTML entities in markup: "<" for "<" + # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string + try: # see #661 + import html + + text = html.unescape(text) + except (ImportError, AttributeError): + from html import parser as HTMLParser + + htmlparser = HTMLParser.HTMLParser() + text = htmlparser.unescape(text) + + return text.lstrip() + + def add_figure(self, name, indentLevel=0): + imageFile = f"img/nxdl/nxdl_{name}.png" + figure_id = f"fig.nxdl_{name}" + file_name = directories.manual_source_sphinxroot() / imageFile + if not file_name.exists(): + return + text = FIGURE_FMT % ( + figure_id, + imageFile, + name, + "80%", + name, + name, + ) + indent = self._indent(indentLevel) + for line in text.splitlines(): + self._print(indent + line) + self._print("\n") + + def pick_nodes_from_xpath(self, parent, path): + return parent.xpath(path, namespaces=self.ns) + + def _print(self, *args, end="\n"): + # TODO: change instances of \t to proper indentation + self._rst_lines.append(" ".join(args) + end) + + +ELEMENT_DICT = { + "attribute": """ +An ``attribute`` element can *only* be a child of a +``field`` or ``group`` element. +It is used to define *attribute* elements to be used and their data types +and possibly an enumeration of allowed values. + +For more details, see: +:ref:`NXDL.data.type.attributeType` + """, + "definition": """ +A ``definition`` element can *only* be used +at the root level of an NXDL specification. +Note: Due to the large number of attributes of the ``definition`` element, +they have been omitted from the figure below. + +For more details, see: +:ref:`NXDL.data.type.definition`, +:ref:`NXDL.data.type.definitionType`, and +:ref:`NXDL.data.type.definitionTypeAttr` + """, + "dimensions": """ +The ``dimensions`` element describes the *shape* of an array. +It is used *only* as a child of a ``field`` element. + +For more details, see: +:ref:`NXDL.data.type.dimensionsType` + """, + "doc": """ +A ``doc`` element can be a child of most NXDL elements. In most cases, the +content of the ``doc`` element will also become part of the NeXus manual. + +:element: {any}: + +In documentation, it may be useful to +use an element that is not directly specified by the NXDL language. +The *any* element here says that one can use any element +at all in a ``doc`` element and NXDL will not process it but pass it through. + +For more details, see: +:ref:`NXDL.data.type.docType` + """, + "enumeration": """ +An ``enumeration`` element can *only* be a child of a +``field`` or ``attribute`` element. +It is used to restrict the available choices to a predefined list, +such as to control varieties in spelling of a controversial word (such as +*metre* vs. *meter*). + +For more details, see: +:ref:`NXDL.data.type.enumerationType` + """, + "field": """ +The ``field`` element provides the value of a named item. Many different attributes +are available to further define the ``field``. Some of the attributes are not +allowed to be used together (such as ``axes`` and ``axis``); see the documentation +of each for details. +It is used *only* as a child of a ``group`` element. + +For more details, see: +:ref:`NXDL.data.type.fieldType` + """, + "choice": """ +A ``choice`` element is used when a named group might take one +of several possible NeXus base classes. Logically, it must +have at least two group children. + +For more details, see: +:ref:`NXDL.data.type.choiceType` + """, + "group": """ +A ``group`` element can *only* be a child of a +``definition`` or ``group`` element. +It describes a common level of organization in a NeXus data file, similar +to a subdirectory in a file directory tree. + +For more details, see: +:ref:`NXDL.data.type.groupType` + """, + "link": """ +.. index:: + single: link target + +A ``link`` element can *only* be a child of a +``definition``, +``field``, or ``group`` element. +It describes the path to the original source of the parent +``definition``, +``field``, or ``group``. + +For more details, see: +:ref:`NXDL.data.type.linkType` + """, + "symbols": """ +A ``symbols`` element can *only* be a child of a ``definition`` element. +It defines the array index symbols to be used when defining arrays as +``field`` elements with common dimensions and lengths. + +For more details, see: +:ref:`NXDL.data.type.symbolsType` + """, +} + + +ELEMENT_PREAMBLE = """ +============================= +NXDL Elements and Field Types +============================= + +The documentation in this section has been obtained directly +from the NXDL Schema file: *nxdl.xsd*. +First, the basic elements are defined in alphabetical order. +Attributes to an element are indicated immediately following the element +and are preceded with an "@" symbol, such as +**@attribute**. +Then, the common data types used within the NXDL specification are defined. +Pay particular attention to the rules for *validItemName* +and *validNXClassName*. + +.. + 2010-11-29,PRJ: + This contains a lot of special case code to lay out the NXDL chapter. + It could be cleaner but that would also involve some cooperation on + anyone who edits nxdl.xsd which is sure to break. The special case ensures + the parts come out in the chosen order. BUT, it is possible that new + items in nxdl.xsd will not automatically go in the manual. + Can this be streamlined with some common methods? + Also, there is probably too much documentation in nxdl.xsd. Obscures the function. + +.. index:: + see:attribute; NXDL attribute + ! single: NXDL elements + +.. _NXDL.elements: + +NXDL Elements +============= + + """ + +DATATYPE_PREAMBLE = """ + +.. _NXDL.data.types.internal: + +NXDL Field Types (internal) +=========================== + +Field types that define the NXDL language are described here. +These data types are defined in the XSD Schema (``nxdl.xsd``) +and are used in various parts of the Schema to define common structures +or to simplify a complicated entry. While the data types are not intended for +use in NXDL specifications, they define structures that may be used in NXDL specifications. + +""" + +DATATYPE_POSTAMBLE = """ +**The** ``xs:string`` **data type** + The ``xs:string`` data type can contain characters, + line feeds, carriage returns, and tab characters. + See https://www.w3schools.com/xml/schema_dtypes_string.asp + for more details. + +**The** ``xs:token`` **data type** + The ``xs:string`` data type is derived from the + ``xs:string`` data type. + + The ``xs:token`` data type also contains characters, + but the XML processor will remove line feeds, carriage returns, tabs, + leading and trailing spaces, and multiple spaces. + See https://www.w3schools.com/xml/schema_dtypes_string.asp + for more details. +""" + + +FIGURE_FMT = """ +.. compound:: + + .. _%s: + + .. figure:: %s + :alt: fig.nxdl/nxdl_%s + :width: %s + + Graphical representation of the NXDL ``%s`` element + + .. Images of NXDL structure are generated from nxdl.xsd source + using the Eclipse XML Schema Editor (Web Tools Platform). Open the nxdl.xsd file and choose the + "Design" tab. Identify the structure to be documented and double-click to expand + as needed to show the detail. Use the XSD > "Export Diagram as Image ..." menu item (also available + as button in top toolbar). + Set the name: "nxdl_%s.png" and move the file into the correct location using + your operating system's commands. Commit the revision to version control. +""" diff --git a/dev_tools/docs/xsd_units.py b/dev_tools/docs/xsd_units.py new file mode 100644 index 0000000000..63d7714289 --- /dev/null +++ b/dev_tools/docs/xsd_units.py @@ -0,0 +1,72 @@ +""" +Read the the NeXus NXDL types specification and find +all the valid types of units. Write a restructured +text (.rst) document for use in the NeXus manual in +the NXDL chapter. +""" + + +from pathlib import Path +from typing import List + +import lxml.etree + +from ..globals.nxdl import XSD_NAMESPACE +from ..utils.types import PathLike + + +def generate_xsd_units_doc( + xsd_file: PathLike, nodeMatchString: str, section: str +) -> List[str]: + xsd_file = str(Path(xsd_file)) + tree = lxml.etree.parse(xsd_file) + + rst_lines = [f".. auto-generated by {__name__} -- DO NOT EDIT\n"] + rst_lines.append("\n") + + rst_lines.append(f".. nodeMatchString : {nodeMatchString}\n") + rst_lines.append("\n") + + db = {} + ns = {"xs": XSD_NAMESPACE} + node_list = tree.xpath("//xs:simpleType", namespaces=ns) + + # get the names of all the types of units + members = [] + for node in node_list: + if node.get("name") == nodeMatchString: + union = node.xpath("xs:union", namespaces=ns) + members = union[0].get("memberTypes", "").split() + + # get the definition of each type of units + for node in node_list: + node_name = node.get("name") + if node_name is None: + continue + if "nxdl:" + node_name in members: + words = node.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] + examples = [] + for example in words.iterchildren(): + nm = example.attrib.get("name") + if nm is not None and nm == "example": + examples.append("``" + example.text + "``") + a = words.text + if len(examples) > 0: + a = " ".join(a.split()) + ",\n\texample(s): " + " | ".join(examples) + db[node_name] = a + + # for item in node.xpath("xs:restriction//xs:enumeration", namespaces=ns): + # key = "%s" % item.get("value") + # words = item.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] + # db[key] = words.text + + # this list is too long to make this a table in latex + # for two columns, a Sphinx fieldlist will do just as well + for key in sorted(db): + rst_lines.append(f".. index:: ! {key} ({section} type)\n\n") # index entry + rst_lines.append(f".. _{key}:\n\n") # cross-reference point + rst_lines.append(f":{key}:\n") + for line in db[key].splitlines(): + rst_lines.append(f" {line}\n") + rst_lines.append("\n") + return rst_lines diff --git a/dev_tools/globals/__init__.py b/dev_tools/globals/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/globals/directories.py b/dev_tools/globals/directories.py new file mode 100644 index 0000000000..9b8498ef9a --- /dev/null +++ b/dev_tools/globals/directories.py @@ -0,0 +1,121 @@ +import os +from pathlib import Path + +from ..utils.types import PathLike + + +def get_source_root() -> Path: + """Root directory of the source code for the documentation + (git repository root by default)""" + if _SOURCE_ROOT is None: + set_source_root(Path(__file__).absolute().parent.parent.parent) + return _SOURCE_ROOT + + +def set_source_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + global _SOURCE_ROOT + _SOURCE_ROOT = root.absolute() + + +def get_nxdl_root() -> Path: + """Root directory of the XSD and NXDL files + (same as the source root by default)""" + if _NXDL_ROOT is None: + set_nxdl_root(get_source_root()) + return _NXDL_ROOT + + +def set_nxdl_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + assert (root / _XSD_FILE_NAME).exists() + global _NXDL_ROOT + _NXDL_ROOT = root.absolute() + + +def get_xsd_file() -> Path: + """The XSD file that defines the NXDL""" + return get_nxdl_root() / _XSD_FILE_NAME + + +def get_xsd_units_file() -> Path: + """The XSD file that defines the units in the NXDL""" + return get_nxdl_root() / _XSD_UNITS_FILE_NAME + + +def get_nxdl_version_file() -> Path: + """The version of the NeXus standard and the NeXus Definition language""" + return get_source_root() / _VERSION_FILE_NAME + + +def get_build_root() -> Path: + """Root directory in which the sources for documentation + building are generated.""" + if _BUILD_ROOT is None: + set_build_root(get_source_root() / "build") + return _BUILD_ROOT + + +def set_build_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + global _BUILD_ROOT + _BUILD_ROOT = root.absolute() + + +def manual_source_root() -> Path: + """Source directory of the NeXus User Manual""" + return get_source_root() / "manual" + + +def manual_build_root() -> Path: + """Build directory of the NeXus User Manual""" + return get_build_root() / "manual" + + +def impatient_source_root() -> Path: + """Source directory of the NeXus Impatient Guide""" + return get_source_root() / "impatient-guide" + + +def impatient_build_root() -> Path: + """Build directory of the NeXus Impatient Guide""" + return get_build_root() / "impatient-guide" + + +def manual_source_sphinxroot() -> Path: + """Sphinx source directory of the NeXus User Manual""" + return manual_source_root() / "source" + + +def manual_build_sphinxroot() -> Path: + """Sphinx source directory for building of the NeXus User Manual""" + return manual_build_root() / "source" + + +def manual_build_staticroot() -> Path: + """Static source directory for building of the NeXus User Manual""" + return manual_build_sphinxroot() / "_static" + + +def nxclass_build_root(nxdl_file: Path) -> Path: + """NeXus class documentation directory for building of the NeXus User Manual""" + root = manual_build_sphinxroot() / "classes" + root /= os.path.relpath(nxdl_file.parent, get_nxdl_root()) + return root + + +def get_rst_filename(nxdl_file: Path) -> Path: + rst_file_name = nxclass_build_root(nxdl_file) + rst_file_name /= nxdl_file.with_suffix("").with_suffix(".rst").name + return rst_file_name + + +_XSD_FILE_NAME = "nxdl.xsd" +_XSD_UNITS_FILE_NAME = "nxdlTypes.xsd" +_VERSION_FILE_NAME = "NXDL_VERSION" +_NXDL_ROOT = None +_BUILD_ROOT = None +_SOURCE_ROOT = None diff --git a/dev_tools/globals/errors.py b/dev_tools/globals/errors.py new file mode 100644 index 0000000000..7b15e9fa60 --- /dev/null +++ b/dev_tools/globals/errors.py @@ -0,0 +1,14 @@ +class XMLSyntaxError(Exception): + """XML file has a syntax error""" + + pass + + +class NXDLSyntaxError(XMLSyntaxError): + """XML file written in NXDL has a syntax error""" + + pass + + +class NXDLParseError(Exception): + """XML file written in NXDL cannot be parsed""" diff --git a/dev_tools/globals/nxdl.py b/dev_tools/globals/nxdl.py new file mode 100644 index 0000000000..eb4013e2cf --- /dev/null +++ b/dev_tools/globals/nxdl.py @@ -0,0 +1,21 @@ +"""Namespace of the schema in which NXDL is defined +(the XSD namespace) and the namespace of the NeXus +class definitions (the NXDL namespace). + +Namespaces are URL's solely to be globally unique. +Do not use these URL's for validation. That's what +"xsi:schemaLocation" is for. Currently this is the +relative URI "../nxdl.xsd" which means validation +can only be done for NXDL files in subdirectories. +""" + +from .directories import get_nxdl_version_file + +XSD_NAMESPACE = "http://www.w3.org/2001/XMLSchema" +NXDL_NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" + + +def get_nxdl_version() -> str: + """The version of the NeXus standard and the NeXus Definition language""" + with open(get_nxdl_version_file(), "r") as fh: + return fh.read().strip() diff --git a/dev_tools/globals/urls.py b/dev_tools/globals/urls.py new file mode 100644 index 0000000000..729b7d1028 --- /dev/null +++ b/dev_tools/globals/urls.py @@ -0,0 +1,2 @@ +REPO_URL = "https://github.com/nexusformat/definitions/blob/main" +MANUAL_URL = "https://manual.nexusformat.org" diff --git a/dev_tools/nxdl/__init__.py b/dev_tools/nxdl/__init__.py new file mode 100644 index 0000000000..33b2684778 --- /dev/null +++ b/dev_tools/nxdl/__init__.py @@ -0,0 +1,10 @@ +"""NeXus classes are defined in the NeXus Definition Language (NXDL). + +The NeXus Definition Language is an XML Schema Definition (XSD). +Each NeXus class is defined in an XML file using the NXDL schema. +""" + +from .discover import find_definition # noqa F401 +from .discover import iter_definitions # noqa F401 +from .syntax import nxdl_schema # noqa F401 +from .syntax import validate_definition # noqa F401 diff --git a/dev_tools/nxdl/discover.py b/dev_tools/nxdl/discover.py new file mode 100644 index 0000000000..6f64425a73 --- /dev/null +++ b/dev_tools/nxdl/discover.py @@ -0,0 +1,30 @@ +from pathlib import Path +from typing import Iterator +from typing import Optional +from typing import Tuple + +from ..globals.directories import get_build_root +from ..globals.directories import get_nxdl_root + + +def iter_definitions(*subdirs: Tuple[str]) -> Iterator[Path]: + """Yield all NeXus class definitions""" + root = get_nxdl_root() + build_root = get_build_root() + for subdir in subdirs: + root /= subdir + for path in sorted(root.rglob("*.nxdl.xml")): + try: + path.relative_to(build_root) + # Skip file in the build root + continue + except ValueError: + yield path + + +def find_definition(nxclass: str, *subdirs: Tuple[str]) -> Optional[Path]: + """Find the definition of a NeXus class""" + s = nxclass.lower() + ".nxdl" + for path in iter_definitions(*subdirs): + if path.stem.lower() == s: + return path diff --git a/dev_tools/nxdl/syntax.py b/dev_tools/nxdl/syntax.py new file mode 100644 index 0000000000..e5b37b657e --- /dev/null +++ b/dev_tools/nxdl/syntax.py @@ -0,0 +1,28 @@ +from typing import Optional + +import lxml.etree + +from ..globals import errors +from ..globals.directories import get_xsd_file +from ..utils.types import PathLike + + +def nxdl_schema() -> lxml.etree.XMLSchema: + return lxml.etree.XMLSchema(lxml.etree.parse(get_xsd_file())) + + +def validate_definition( + xml_file_name: PathLike, + xml_schema: Optional[lxml.etree.XMLSchema] = None, +): + xml_file_name = str(xml_file_name) + try: + xml_tree = lxml.etree.parse(xml_file_name) + except lxml.etree.XMLSyntaxError: + raise errors.XMLSyntaxError(xml_file_name) + if xml_schema is None: + xml_schema = nxdl_schema() + try: + xml_schema.assertValid(xml_tree) + except lxml.etree.DocumentInvalid: + raise errors.NXDLSyntaxError(xml_file_name) diff --git a/dev_tools/tests/__init__.py b/dev_tools/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/tests/test_docs.py b/dev_tools/tests/test_docs.py new file mode 100644 index 0000000000..f0a8bca984 --- /dev/null +++ b/dev_tools/tests/test_docs.py @@ -0,0 +1,52 @@ +import pytest + +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..docs import XSDDocGenerator +from ..docs import nxdl_indices +from ..globals.directories import get_xsd_file +from ..nxdl import iter_definitions + + +@pytest.fixture(scope="module") +def doc_generator(): + return NXClassDocGenerator() + + +@pytest.fixture(scope="module") +def anchor_registry(): + return AnchorRegistry() + + +@pytest.fixture(scope="module") +def anchor_registry_write(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("registry") + reg = AnchorRegistry(output_path=tmpdir) + yield reg + reg.write() + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_generate_doc(nxdl_file, doc_generator): + assert doc_generator(nxdl_file) + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_anchor_list(nxdl_file, doc_generator, anchor_registry): + assert doc_generator(nxdl_file, anchor_registry=anchor_registry) + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_anchor_write_list(nxdl_file, doc_generator, anchor_registry_write): + assert doc_generator(nxdl_file, anchor_registry=anchor_registry_write) + + +def test_nxdl_indices(): + sections = nxdl_indices() + expected = {"base_classes", "applications", "contributed_definitions"} + assert set(sections) == expected + + +def test_xsd_generate_doc(): + generator = XSDDocGenerator() + assert generator(get_xsd_file()) diff --git a/dev_tools/tests/test_nxdl.py b/dev_tools/tests/test_nxdl.py new file mode 100644 index 0000000000..bdeafe124a --- /dev/null +++ b/dev_tools/tests/test_nxdl.py @@ -0,0 +1,30 @@ +import pytest + +from ..nxdl import find_definition +from ..nxdl import iter_definitions +from ..nxdl import nxdl_schema +from ..nxdl import validate_definition + + +def test_iter_definitions(): + all_files = set(iter_definitions()) + assert all_files + base_files = set(iter_definitions("base_classes")) + assert base_files + assert not (base_files - all_files) + + +def test_find_definition(): + assert find_definition("NXroot") + assert not find_definition("NXwrong") + assert not find_definition("NXroot", "applications") + + +@pytest.fixture(scope="module") +def xml_schema(): + return nxdl_schema() + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_syntax(nxdl_file, xml_schema): + validate_definition(nxdl_file, xml_schema) diff --git a/dev_tools/utils/__init__.py b/dev_tools/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/utils/copy.py b/dev_tools/utils/copy.py new file mode 100644 index 0000000000..472c4b4ad0 --- /dev/null +++ b/dev_tools/utils/copy.py @@ -0,0 +1,50 @@ +import shutil +from pathlib import Path +from typing import List +from typing import Tuple + +import requests + +from ..globals import directories + + +def copyfile(from_path: Path, to_path: Path) -> None: + print("copy", from_path) + print(" ->", to_path) + shutil.copyfile(from_path, to_path) + + +def copydir(from_path: Path, to_path: Path) -> None: + print("copy", from_path) + print(" ->", to_path) + shutil.copytree(from_path, to_path, dirs_exist_ok=True) + + +def download(url: str, to_path: Path) -> None: + print("download", url) + print(" ->", to_path) + r = requests.get(url) + with open(to_path, "wb") as fh: + fh.write(r.content) + + +def copy_files(files: List[Tuple[str, str, bool]]) -> None: + source_root = directories.get_source_root() + build_root = directories.get_build_root() + for from_subname, to_subname, overwrite in files: + to_path = build_root / to_subname + if overwrite or not to_path.exists(): + from_path = source_root / from_subname + copyfile(from_path, to_path) + else: + print("already exists", to_path) + + +def download_files(urls: List[Tuple[str, str, bool]]) -> None: + build_root = directories.get_build_root() + for url, subname, overwrite in urls: + to_path = build_root / subname + if overwrite or not to_path.exists(): + download(url, to_path) + else: + print("already exists", to_path) diff --git a/dev_tools/utils/diff.py b/dev_tools/utils/diff.py new file mode 100644 index 0000000000..7269ab1a18 --- /dev/null +++ b/dev_tools/utils/diff.py @@ -0,0 +1,28 @@ +import difflib +import tempfile +from pathlib import Path +from typing import List + + +def diff_ascii(src_file: Path, new_content: List[str], dest_file: Path) -> None: + """`new_content` is the parsed content of `src_file` to be compared + with the content of `dest_file`. + """ + if dest_file.exists(): + with open(dest_file, "r") as fh: + original_content = list(fh) + else: + original_content = list() + + with tempfile.TemporaryFile("w+") as fh: + fh.writelines(new_content) + fh.seek(0) + new_content = list(fh) + + for line in difflib.unified_diff( + original_content, + new_content, + fromfile=str(dest_file), + tofile=str(src_file), + ): + print(line) diff --git a/dev_tools/utils/types.py b/dev_tools/utils/types.py new file mode 100644 index 0000000000..a2f5bbf6ff --- /dev/null +++ b/dev_tools/utils/types.py @@ -0,0 +1,4 @@ +import os +from typing import Union + +PathLike = Union[str, os.PathLike] diff --git a/impatient-guide/Makefile b/impatient-guide/Makefile deleted file mode 100644 index fa3c25dcf6..0000000000 --- a/impatient-guide/Makefile +++ /dev/null @@ -1,166 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - touch $(BUILDDIR)/html/.nojekyll - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/NeXusfortheImpatient.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/NeXusfortheImpatient.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/NeXusfortheImpatient" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/NeXusfortheImpatient" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -PDF_version: latexpdf - cp $(BUILDDIR)/latex/NXImpatient.pdf ./ - -EPUB_version: epub - cp $(BUILDDIR)/epub/NeXusfortheImpatient.epub ./NXImpatient.epub - -HTML_version:: PDF_version html - -all: PDF_version HTML_version - -rebuild: clean all diff --git a/manual/Makefile b/manual/Makefile deleted file mode 100644 index 06a3752a77..0000000000 --- a/manual/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = NeXusManual -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# # TODO: where to add this in next line -# # https://github.com/nexusformat/definitions/issues/659 -# # LATEXOPTS="--interaction=nonstopmode" -# latexpdf :: Makefile -# @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - $(MAKE) clean -C $(SOURCEDIR) diff --git a/manual/source/Makefile b/manual/source/Makefile deleted file mode 100755 index da138be897..0000000000 --- a/manual/source/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - - -all :: types.table units.table nxdl_desc.rst subdirs - -types.table: ../../utils/types2rst.py ../../utils/units2rst.py ../../nxdlTypes.xsd - $(PYTHON) -u ../../utils/types2rst.py ../../nxdlTypes.xsd > $@ - -units.table: ../../utils/units2rst.py ../../nxdlTypes.xsd - $(PYTHON) -u ../../utils/units2rst.py ../../nxdlTypes.xsd > $@ - -nxdl_desc.rst: ../../utils/nxdl_desc2rst.py ../../nxdl.xsd - $(PYTHON) -u ../../utils/nxdl_desc2rst.py ../../nxdl.xsd > $@ - -SUBDIRS = classes - -.PHONY: subdirs $(SUBDIRS) - -subdirs: $(SUBDIRS) - -$(SUBDIRS): - $(MAKE) -C $@ - -clean: - $(RM) types.table units.table nxdl_desc.rst - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done diff --git a/manual/source/classes/Makefile b/manual/source/classes/Makefile deleted file mode 100755 index be5f8bf8fc..0000000000 --- a/manual/source/classes/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -SUBDIRS = base_classes applications contributed_definitions - -.PHONY: subdirs $(SUBDIRS) clean - -subdirs: $(SUBDIRS) - -$(SUBDIRS): - $(MAKE) -C $@ - -clean: - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done diff --git a/manual/source/classes/applications/Makefile b/manual/source/classes/applications/Makefile deleted file mode 100644 index 2d6bc33d2e..0000000000 --- a/manual/source/classes/applications/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/applications -SUBDIRS = canSAS - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to applications/index.rst" - $(PYTHON) $(NXDLSUMMARY) applications - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - $(RM) -rf $(SUBDIRS) - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/manual/source/classes/base_classes/Makefile b/manual/source/classes/base_classes/Makefile deleted file mode 100644 index 3b124728c0..0000000000 --- a/manual/source/classes/base_classes/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/base_classes - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to base_classes/index.rst" - $(PYTHON) $(NXDLSUMMARY) base_classes - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/manual/source/classes/contributed_definitions/Makefile b/manual/source/classes/contributed_definitions/Makefile deleted file mode 100644 index 6ddbadb18c..0000000000 --- a/manual/source/classes/contributed_definitions/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/contributed_definitions -SUBDIRS = container - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to contributed_definitions/index.rst" - $(PYTHON) $(NXDLSUMMARY) contributed_definitions - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - $(RM) -rf $(SUBDIRS) - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/requirements.txt b/requirements.txt index 070e65c74f..815406521c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,14 @@ +# Prepare for Documentation lxml pyyaml -Sphinx + +# Documentation building +sphinx>=5 + +# Testing +pytest + +# Code style and auto-formatting +black>=22.3 +flake8>=4 +isort>=5.10 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000000..a49eb0be12 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html + +# E501 (line too long) ignored for now +# E203 and W503 incompatible with black formatting +[flake8] +ignore = E501, E203, W503 +max-line-length = 88 + +[isort] +profile = black +force_single_line = true \ No newline at end of file diff --git a/utils/build_preparation.py b/utils/build_preparation.py deleted file mode 100644 index ecc5876e0e..0000000000 --- a/utils/build_preparation.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python - -""" -Copy all resources for out-of-source documentation build - -Since we provide a build for Linux, MacOSX, and Windows, -this tool must be multiplatform. If only for Linux -(and possibly MacOSX), it might be possible to use a form of:: - - cp -a ../base_classes ./ - cp -a ../applications ./ - cp -a ../contributed_definitions ./ - cp -a ../manual ./ - ... - -Here, we identify and copy all resources to build. -The target directory is assumed to be the current directory. - -""" - -# Re-run this code to bring in any changed files (for incremental build) -# Be sure to properly specify the source and target directories. - -import os, sys -from local_utilities import replicate - - -MTIME_TOLERANCE = 0.001 # ignore mtime differences <= 1 ms -ROOT_DIR_EXPECTED_RESOURCES = { - "files": """COPYING LGPL.txt Makefile NXDL_VERSION - nxdl.xsd nxdlTypes.xsd README.md - """.split(), - "subdirs": """applications base_classes contributed_definitions manual - package utils www impatient-guide - """.split(), -} -REPLICATED_RESOURCES = """ - LGPL.txt Makefile nxdl.xsd nxdlTypes.xsd NXDL_VERSION - base_classes applications contributed_definitions - manual utils impatient-guide -""".split() - - -def mtime_size(filename): - """get the modification time and size of the given item""" - file_status = os.stat(filename) - return file_status.st_mtime, file_status.st_size - - -def standardize_name(path, resource_name): - """always use the absolute path to the filesystem resource""" - return os.path.abspath(os.path.join(path, resource_name)) - - -def identical(source, target): - """compare if the resource is the same on both paths""" - if not os.path.exists(target): - return False - s_mtime, s_size = mtime_size(source) - t_mtime, t_size = mtime_size(target) - return abs(s_mtime - t_mtime) <= MTIME_TOLERANCE and s_size == t_size - - -def get_source_items(resources, source_path): - """walk the source_path directories accumulating files to be checked""" - file_list = [] - path_list = [] - for path in sorted(resources): - source = standardize_name(source_path, path) - if os.path.isfile(source): - file_list.append(source) - else: - for root, dirs, files in os.walk(source): - path_list.append(root) - file_list = file_list + [os.path.join(root, _) for _ in files] - return path_list, file_list - - -def is_definitions_directory(basedir): - """test if ``basedir`` is a NeXus definitions directory""" - # look for the expected files and subdirectories in the root directory - for item_list in ROOT_DIR_EXPECTED_RESOURCES.values(): - for item in item_list: - if not os.path.exists(os.path.join(basedir, item)): - return False - return True - - -def qualify_inputs(source_dir, target_path): - """raise error if this program cannot continue, based on the inputs""" - if not os.path.exists(source_dir): - raise RuntimeError("Cannot find " + source_dir) - - if not os.path.isdir(source_dir): - raise RuntimeError("Not a directory: " + source_dir) - - if not is_definitions_directory(source_dir): - msg = "Not a NeXus definitions root directory " + source_dir - raise RuntimeError(msg) - - if source_dir == target_path: - msg = "Source and target directories cannot be the same" - raise RuntimeError(msg) - - -def command_args(): - """get the command-line arguments, handle syntax errors""" - import argparse - - doc = __doc__.strip().splitlines()[0] - parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "defs_dir", action="store", help="path to NeXus definitions root directory" - ) - parser.add_argument( - "build_dir", - action="store", - default=None, - nargs="?", - help="path to target directory (default: current directory)", - ) - return parser.parse_args() - - -def update(source_path, target_path): - """ - duplicate directory from source_path to target_path - - :param source_path str: source directory (NeXus definitions dir) - :param target_path str: target directory is specified for build product - """ - # TODO: what about file items in target_path that are not in source_path? - source_path = os.path.abspath(source_path) - target_path = os.path.abspath(target_path) - qualify_inputs(source_path, target_path) - - paths, files = get_source_items(REPLICATED_RESOURCES, source_path) - print("source has %d directories and %d files" % (len(paths), len(files))) - - # create all the directories / subdirectories - for source in sorted(paths): - relative_name = source[len(source_path) :].lstrip(os.sep) - target = standardize_name(target_path, relative_name) - if not os.path.exists(target): - print("create directory %s" % target) - os.mkdir(target, os.stat(source_path).st_mode) - # check if the files need to be updated - for source in sorted(files): - relative_name = source[len(source_path) :].lstrip(os.sep) - target = standardize_name(target_path, relative_name) - if not identical(source, target): - print("update file %s" % target) - replicate(source, target) - - -def main(): - """ - standard command-line processing - - source directory (NeXus definitions dir) named as command line argument - target directory is specified (or defaults to present working directory) - """ - cli = command_args() - source_path = os.path.abspath(cli.defs_dir) - target_path = cli.build_dir or os.path.abspath(os.getcwd()) - update(source_path, target_path) - - -def __developer_build_setup__(): - """for use with source-code debugger ONLY""" - import shutil - - # sys.argv.append('-h') - os.chdir("../") - os.chdir("build") - sys.argv.append("..") - - -if __name__ == "__main__": - # __developer_build_setup__() - main() - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2015 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/dev_nxdl2rst.py b/utils/dev_nxdl2rst.py deleted file mode 100755 index 6bbe3e2bf1..0000000000 --- a/utils/dev_nxdl2rst.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -""" -Developers: use this code to develop and test nxdl2rst.py -""" - -# testing: -# cd /tmp -# mkdir out -# /G/nx-def/utils/nxdl2rst.py /G/nx-def/applications/NXsas.nxdl.xml > nxsas.rst && sphinx-build . out -# then point browser to file:///tmp/out/nxsas.html - - -import nxdl2rst -import os -import sys - - -# find the directory of this python file -BASEDIR = os.path.dirname(__file__) - - -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXarchive.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXcanSAS.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXmx.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXsas.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXcrystal.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXentry.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXobject.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXroot.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXuser.nxdl.xml') -nxdl = os.path.join(BASEDIR, "..", "applications", "NXarpes.nxdl.xml") -# nxdl = os.path.join(BASEDIR, '..', 'contributed_definitions', 'NXmagnetic_kicker.nxdl.xml') - - -if len(sys.argv) == 1: - sys.argv.append(nxdl) -elif len(sys.argv) > 1: - sys.argv[1] = nxdl - -nxdl2rst.main() diff --git a/utils/dev_units2rst.py b/utils/dev_units2rst.py deleted file mode 100755 index b43d05f29a..0000000000 --- a/utils/dev_units2rst.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -""" -Developers: use this code to develop and test nxdl2rst.py -""" - -import sys -from units2rst import worker - - -# sys.argv.append("../nxdlTypes.xsd") -sys.argv.append("nxdlTypes.xsd") -worker("anyUnitsAttr") diff --git a/utils/local_utilities.py b/utils/local_utilities.py deleted file mode 100644 index 54a3db6313..0000000000 --- a/utils/local_utilities.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -""" -Common code for NeXus definitions Python tools - -====================== ================================== -tool description -====================== ================================== -:meth:`printf` formatted print without newline -:meth:`mtime` return file modification time -:meth:`replicate` copy directory stack or file -:meth:`replicate_tree` copy directory stack -====================== ================================== - -""" - -import os -import shutil - - -def mtime(file_name): - """return file modification time""" - return os.stat(file_name)[os.stat.ST_MTIME] - - -def replicate(source, target): - """ - for directories or files: copy ``source`` to ``target``, replaces ``target`` - - :param str source: path to source resource - :param str target: path to target location - """ - if os.path.isfile(source): - shutil.copy2(source, target) - elif os.path.isdir(source): - replicate_tree(source, target) - else: - msg = "Do not know how to copy (skipped): " + source - raise RuntimeWarning(msg) - - -def replicate_tree(source, target): - """ - for directories: copy ``source`` to ``target``, replaces ``target`` - - :param str source: path to source resource (a directory) - :param str target: path to target location (a directory) - """ - if os.path.exists(source): - if os.path.exists(target): - shutil.rmtree(target, ignore_errors=True) - shutil.copytree(source, target) - else: - raise RuntimeError("Directory not found: " + source) - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/nxdl2rst.py b/utils/nxdl2rst.py deleted file mode 100755 index 2493046525..0000000000 --- a/utils/nxdl2rst.py +++ /dev/null @@ -1,814 +0,0 @@ -#!/usr/bin/env python - -""" -Read the NeXus NXDL class specification and describe it. -Write a restructured text (.rst) document for use in the NeXus manual in -the NeXus NXDL Classes chapter. -""" - -# testing: see file dev_nxdl2rst.py - -from collections import OrderedDict -from html import parser as HTMLParser -import datetime -import json -import lxml.etree -import os -import pathlib -import re -import sys -import yaml -from local_utilities import replicate - - -INDENTATION_UNIT = " " -use_application_defaults = None -repo_root_path = pathlib.Path(__file__).parent.parent -WRITE_ANCHOR_REGISTRY = False -HTML_ROOT = "https://github.com/nexusformat/definitions/blob/main" -MANUAL_ROOT = "https://manual.nexusformat.org/" - - -class AnchorRegistry: - """Document the NXDL vocabulary.""" - - def __init__(self) -> None: - path = repo_root_path / "manual" / "source" / "_static" - base = "nxdl_vocabulary" - self.html_file = path / f"{base}.html" - self.txt_file = path / f"{base}.txt" - self.json_file = path / f"{base}.json" - self.yaml_file = path / f"{base}.yml" - self.registry = self._read() - self.local_anchors = [] # anchors from current NXDL file - self.nxdl_file = None - self.nxdl_subdir = None - - @property - def all_anchors(self): - result = [] - for v in self.registry.values(): - result += list(v.keys()) - return result - - def add(self, anchor): - if anchor not in self.local_anchors: - self.local_anchors.append(anchor) - - key = self.key_from_anchor(anchor) - - if key not in self.registry: - self.registry[key] = {} - - reg = self.registry[key] - if anchor not in reg: - hanchor = self._html_anchor(anchor) - fnxdl = "/".join(pathlib.Path(self.nxdl_file).parts[-2:]).split(".")[0] - url = f"{MANUAL_ROOT}classes/{self.nxdl_subdir}/{fnxdl}.html{hanchor}" - reg[anchor] = dict(term=anchor, html=hanchor, url=url,) - - def key_from_anchor(self, anchor): - key = anchor.lower().split("/")[-1].split("@")[-1].split("-")[0] - if "@" in anchor: - # restore preceding "@" symbol - key = "@" + key - return key - - def write(self): - # fmt: off - version = open(repo_root_path / "NXDL_VERSION", "r").read().strip() - contents = dict( - _metadata=dict( - datetime=datetime.datetime.utcnow().isoformat(), - title="NeXus NXDL vocabulary.", - subtitle=( - "Anchors for all NeXus fields," - " groups, attributes, and links." - ), - version=version, - ), - terms=self.registry, - ) - # fmt: on - - self._write_yaml(contents) - self._write_json(contents) - self._write_txt() - self._write_html(contents) - - def _html_anchor(self, anchor): - """ - Create (internal hyperlink target for) HTML anchor from reST anchor. - - Example: - - * reST anchor: /NXcanSAS/ENTRY/TRANSMISSION_SPECTRUM@timestamp-attribute - * HTML anchor: #nxcansas-entry-transmission-spectrum-timestamp-attribute - """ - html_anchor = ( - anchor.lower() - .lstrip("/") - .replace("_", "-") - .replace("@", "-") - .replace("/", "-") - ) - return f"#{html_anchor}" - - def _read(self): - """The YAML file will record anchors (terms) from all NXDL files.""" - registry = None - if self.yaml_file.exists(): - contents = yaml.load(open(self.yaml_file, "r").read(), Loader=yaml.Loader) - if contents is not None: - registry = contents.get("terms") - return registry or {} - - def _write_html(self, contents): - """Write the anchors to an HTML file.""" - root = lxml.etree.Element("html") - body = lxml.etree.SubElement(root, "body") - title = lxml.etree.SubElement(body, "h1") - subtitle = lxml.etree.SubElement(body, "em") - - title.text = contents["_metadata"]["title"].strip(".") - subtitle.text = contents["_metadata"]["subtitle"].strip(".") - vocab_list = lxml.etree.SubElement(body, "h2") - vocab_list.text = "NXDL Vocabulary" - - p = lxml.etree.SubElement(body, "p") - p.text = "This content is also available in these formats: " - for ext in "json txt yml".split(): - a = lxml.etree.SubElement(p, "a") - a.attrib["href"] = f"{MANUAL_ROOT}_static/{self.txt_file.stem}.{ext}" - a.text = f" {ext}" - - dl = lxml.etree.SubElement(body, "dl") - for term, termlist in sorted(contents["terms"].items()): - dterm = lxml.etree.SubElement(dl, "dt") - dterm.text = term - for _, itemdict in sorted(termlist.items()): - ddef = lxml.etree.SubElement(dterm, "dd") - a = lxml.etree.SubElement(ddef, "a") - a.attrib["href"] = itemdict["url"] - a.text = itemdict["term"] - - lxml.etree.SubElement(body, "hr") - - foot = lxml.etree.SubElement(body, "p") - foot_em = lxml.etree.SubElement(foot, "em") - foot_em.text = f"written: {contents['_metadata']['datetime']}" - - html = lxml.etree.tostring(root, pretty_print=True).decode() - with open(self.html_file, "w") as f: - f.write(html) - f.write("\n") - - def _write_json(self, contents): - with open(self.json_file, "w") as f: - json.dump(contents, f, indent=4, sort_keys=True) - f.write("\n") - - def _write_txt(self): - """Compendium (dump the list of all known anchors in raw form).""" - terms = self.all_anchors - with open(self.txt_file, "w") as f: - f.write("\n".join(sorted(terms))) - f.write("\n") - - def _write_yaml(self, contents): - with open(self.yaml_file, "w") as f: - yaml.dump(contents, f) - - -anchor_registry = AnchorRegistry() - - -def printAnchorList(): - """Print the list of hypertext anchors.""" - - def sorter(key): - return key.lower() - - if len(anchor_registry.local_anchors) > 0: - if WRITE_ANCHOR_REGISTRY: - # ONLY in the build directory - anchor_registry.write() - - print("") - print("Hypertext Anchors") - print("-----------------\n") - print( - "List of hypertext anchors for all groups, fields,\n" - "attributes, and links defined in this class.\n\n" - ) - # fmt: off - rst = [ - f"* :ref:`{ref} <{ref}>`" - for ref in sorted(anchor_registry.local_anchors, key=sorter) - ] - # fmt: on - print("\n".join(rst)) - - -def fmtTyp(node): - typ = node.get("type", ":ref:`NX_CHAR `") # per default - if typ.startswith("NX_"): - typ = ":ref:`%s <%s>`" % (typ, typ) - return typ - - -def fmtUnits(node): - units = node.get("units", "") - if not units: - return "" - if units.startswith("NX_"): - units = "\ :ref:`%s <%s>`" % (units, units) - return " {units=%s}" % units - - -def getDocBlocks(ns, node): - docnodes = node.xpath("nx:doc", namespaces=ns) - if docnodes is None or len(docnodes) == 0: - return "" - if len(docnodes) > 1: - raise Exception( - "Too many doc elements: line %d, %s" - % (node.sourceline, os.path.split(node.base)[1]) - ) - docnode = docnodes[0] - - # be sure to grab _all_ content in the documentation - # it might look like XML - s = lxml.etree.tostring( - docnode, pretty_print=True, method="c14n", with_comments=False - ).decode("utf-8") - m = re.search(r"^]*>\n?(.*)\n?$", s, re.DOTALL) - if not m: - raise Exception("unexpected docstring [%s] " % s) - text = m.group(1) - - # substitute HTML entities in markup: "<" for "<" - # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string - htmlparser = HTMLParser.HTMLParser() - try: # see #661 - import html - - text = html.unescape(text) - except (ImportError, AttributeError): - text = htmlparser.unescape(text) - - # Blocks are separated by whitelines - blocks = re.split("\n\s*\n", text) - if len(blocks) == 1 and len(blocks[0].splitlines()) == 1: - return [blocks[0].rstrip().lstrip()] - - # Indentation must be given by first line - m = re.match(r"(\s*)(\S+)", blocks[0]) - if not m: - return [""] - indent = m.group(1) - - # Remove common indentation as determined from first line - if indent == "": - raise Exception( - "Missing initial indentation in of %s [%s]" - % (node.get("name"), blocks[0]) - ) - - out_blocks = [] - for block in blocks: - lines = block.rstrip().splitlines() - out_lines = [] - for line in lines: - if line[: len(indent)] != indent: - raise Exception( - 'Bad indentation in of %s [%s]: expected "%s" found "%s".' - % ( - node.get("name"), - block, - re.sub(r"\t", "\\\\t", indent), - re.sub(r"\t", "\\\\t", line), - ) - ) - out_lines.append(line[len(indent) :]) - out_blocks.append("\n".join(out_lines)) - return out_blocks - - -def getDocLine(ns, node): - blocks = getDocBlocks(ns, node) - if len(blocks) == 0: - return "" - if len(blocks) > 1: - raise Exception("Unexpected multi-paragraph doc [%s]" % "|".join(blocks)) - return re.sub(r"\n", " ", blocks[0]) - - -def get_minOccurs(node): - """ - get the value for the ``minOccurs`` attribute - - :param obj node: instance of lxml.etree._Element - :returns str: value of the attribute (or its default) - """ - # TODO: can we improve on the default by examining nxdl.xsd? - minOccurs_default = {True: "1", False: "0"}[use_application_defaults] - minOccurs = node.get("minOccurs", minOccurs_default) - return minOccurs - - -def get_required_or_optional_text(node): - """ - make clear if a reported item is required or optional - - :param obj node: instance of lxml.etree._Element - :returns: formatted text - """ - tag = node.tag.split("}")[-1] - nm = node.get("name") - if tag in ("field", "group"): - optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) - minOccurs = get_minOccurs(node) - if recommended: - optional_text = "(recommended) " - elif minOccurs in ("0", 0) or optional: - optional_text = "(optional) " - elif minOccurs in ("1", 1): - optional_text = "(required) " - else: - # this is unexpected and remarkable - # TODO: add a remark to the log - optional_text = "(``minOccurs=%s``) " % str(minOccurs) - elif tag in ("attribute",): - optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) - optional_text = {True: "(optional) ", False: "(required) "}[optional] - if recommended: - optional_text = "(recommended) " - else: - optional_text = "(unknown tag: " + str(tag) + ") " - return optional_text - - -def analyzeDimensions(ns, parent): - """These are the different dimensions that can occur: - - 1. Fixed rank - - - - - - - - 2. Variable rank because of optional dimensions - - - - - - - - - 3. Variable rank because no dimensions specified - - - - - The legacy way of doing this (still supported) - - - - - - 4. Rank and dimensions equal to that of another field called `field_name` - - - - - """ - node_list = parent.xpath("nx:dimensions", namespaces=ns) - if len(node_list) != 1: - return "" - node = node_list[0] - node_list = node.xpath("nx:dim", namespaces=ns) - - dims = [] - optional = False - for subnode in node_list: - # Dimension index (starts from index 1) - index = subnode.get("index", "") - if not index.isdigit(): - raise RuntimeError("A dimension must have an index") - index = int(index) - if index == 0: - # No longer needed: legacy way to specify that the - # rank is variable - continue - - # Expand dimensions when needed - index -= 1 - nadd = max(index - len(dims) + 1, 0) - if nadd: - dims += ["."] * nadd - - # Dimension symbol - dim = subnode.get("value") # integer or symbol from the table - if not dim: - ref = subnode.get("ref") - if ref: - return " (Rank: same as field %s, Dimensions: same as field %s)" % ( - ref, - ref, - ) - dim = "." # dimension has no symbol - - # Dimension might be optional - if subnode.get("required", "true").lower() == "false": - optional = True - elif optional: - raise RuntimeError( - "A required dimension cannot come after an optional dimension" - ) - if optional: - dim = "[%s]" % dim - - dims[index] = dim - - # When the rank is missing, set to the number of dimensions when - # there are dimensions specified and none of them are optional. - ndims = len(dims) - rank = node.get("rank", None) - if rank is None and not optional and ndims: - rank = str(ndims) - - # Validate rank and dimensions - rank_is_fixed = rank and rank.isdigit() - if optional and rank_is_fixed: - raise RuntimeError("A fixed rank cannot have optional dimensions") - if rank_is_fixed and ndims and int(rank) != ndims: - raise RuntimeError("The rank and the number of dimensions do not correspond") - - # Omit rank and/or dimensions when not specified - if rank and dims: - dims = ", ".join(dims) - return " (Rank: %s, Dimensions: [%s])" % (rank, dims) - elif rank: - return " (Rank: %s)" % rank - elif dims: - dims = ", ".join(dims) - return " (Dimensions: [%s])" % dims - return "" - - -def hyperlinkTarget(parent_path, name, nxtype): - """Return internal hyperlink target for HTML anchor.""" - if nxtype == "attribute": - sep = "@" - else: - sep = "/" - target = "%s%s%s-%s" % (parent_path, sep, name, nxtype) - anchor_registry.add(target) - return ".. _%s:\n" % target - - -def printEnumeration(indent, ns, parent): - node_list = parent.xpath("nx:item", namespaces=ns) - if len(node_list) == 0: - return "" - - if len(node_list) == 1: - print(f"{indent}Obligatory value:", end="") - else: - print(f"{indent}Any of these values:", end="") - - docs = OrderedDict() - for item in node_list: - name = item.get("value") - docs[name] = getDocLine(ns, item) - - ENUMERATION_INLINE_LENGTH = 60 - - def show_as_typed_text(msg): - return "``%s``" % msg - - oneliner = " | ".join(map(show_as_typed_text, docs.keys())) - if any(doc for doc in docs.values()) or len(oneliner) > ENUMERATION_INLINE_LENGTH: - # print one item per line - print("\n") - for name, doc in docs.items(): - print(f"{indent} * {show_as_typed_text(name)}", end="") - if doc: - print(f": {doc}", end="") - print("\n") - else: - # print all items in one line - print(f" {oneliner}") - print("") - - -def printDoc(indent, ns, node, required=False): - blocks = getDocBlocks(ns, node) - if len(blocks) == 0: - if required: - raise Exception("No documentation for: " + node.get("name")) - print("") - else: - for block in blocks: - for line in block.splitlines(): - print(f"{indent}{line}") - print() - - -def printAttribute(ns, kind, node, optional, indent, parent_path): - name = node.get("name") - index_name = name - print(f"{indent}" f"{hyperlinkTarget(parent_path, name, 'attribute')}") - print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") - print(f"{indent}**@{name}**: {optional}{fmtTyp(node)}{fmtUnits(node)}\n") - printDoc(indent + INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) - - -def printIfDeprecated(ns, node, indent): - deprecated = node.get("deprecated", None) - if deprecated is not None: - print(f"\n{indent}.. index:: deprecated\n") - print(f"\n{indent}**DEPRECATED**: {deprecated}\n") - - -def printFullTree(ns, parent, name, indent, parent_path): - """ - recursively print the full tree structure - - :param dict ns: dictionary of namespaces for use in XPath expressions - :param lxml_element_node parent: parent node to be documented - :param str name: name of elements, such as NXentry/NXuser - :param indent: to keep track of indentation level - :param parent_path: NX class path of parent nodes - """ - - for node in parent.xpath("nx:field", namespaces=ns): - name = node.get("name") - index_name = name - dims = analyzeDimensions(ns, node) - - optional_text = get_required_or_optional_text(node) - print(f"{indent}{hyperlinkTarget(parent_path, name, 'field')}") - print(f"{indent}.. index:: {index_name} (field)\n") - print( - f"{indent}**{name}**: " - f"{optional_text}" - f"{fmtTyp(node)}" - f"{dims}" - f"{fmtUnits(node)}" - "\n" - ) - - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) - - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) - - for subnode in node.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, - "field", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) - - for node in parent.xpath("nx:group", namespaces=ns): - name = node.get("name", "") - typ = node.get("type", "untyped (this is an error; please report)") - - optional_text = get_required_or_optional_text(node) - if typ.startswith("NX"): - if name == "": - name = typ.lstrip("NX").upper() - typ = ":ref:`%s`" % typ - hTarget = hyperlinkTarget(parent_path, name, "group") - target = hTarget.replace(".. _", "").replace(":\n", "") - # TODO: https://github.com/nexusformat/definitions/issues/1057 - print(f"{indent}{hTarget}") - print(f"{indent}**{name}**: {optional_text}{typ}\n") - - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) - - for subnode in node.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, - "group", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) - - nodename = "%s/%s" % (name, node.get("type")) - printFullTree( - ns, node, nodename, indent + INDENTATION_UNIT, parent_path + "/" + name - ) - - for node in parent.xpath("nx:link", namespaces=ns): - name = node.get("name") - print(f"{indent}{hyperlinkTarget(parent_path, name, 'link')}") - print( - f"{indent}**{name}**: " - ":ref:`link` " - f"(suggested target: ``{node.get('target')}``" - "\n" - ) - printDoc(indent + INDENTATION_UNIT, ns, node) - - -def print_rst_from_nxdl(nxdl_file): - """ - print restructured text from the named .nxdl.xml file - """ - global use_application_defaults - - # parse input file into tree - tree = lxml.etree.parse(nxdl_file) - - # The following URL is outdated, but that doesn't matter; - # it won't be accessed; it's just an arbitrary namespace name. - # It only needs to match the xmlns attribute in the NXDL files. - NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" - ns = {"nx": NAMESPACE} - - root = tree.getroot() - name = root.get("name") - title = name - parent_path = "/" + name # absolute path of parent nodes, no trailing / - if len(name) < 2 or name[0:2] != "NX": - raise Exception('Unexpected class name "%s"; does not start with NX' % (name)) - lexical_name = name[2:] # without padding 'NX', for indexing - - category = root.attrib["category"] - - # Pass these terms to construct the full URL - anchor_registry.nxdl_file = nxdl_file - nxdl_subdir = os.path.basename(os.path.dirname(os.path.abspath(nxdl_file))) - anchor_registry.nxdl_subdir = nxdl_subdir - - listing_category = { - "base": "base class", - "application": "application definition", - }[category] - - use_application_defaults = category == "application" - - # print ReST comments and section header - print( - f".. auto-generated by script {sys.argv[0]} " - f"from the NXDL source {sys.argv[1]}" - ) - print("") - print(".. index::") - print(f" ! {name} ({listing_category})") - print(f" ! {lexical_name} ({listing_category})") - print(f" see: {lexical_name} ({listing_category}); {name}") - print("") - print(f".. _{name}:\n") - print("=" * len(title)) - print(title) - print("=" * len(title)) - - # print category & parent class - extends = root.get("extends") - if extends is None: - extends = "none" - else: - extends = ":ref:`%s`" % extends - - print("") - print("**Status**:\n") - print(f" {listing_category.strip()}, extends {extends}") - - printIfDeprecated(ns, root, "") - - # print official description of this class - print("") - print("**Description**:\n") - printDoc(INDENTATION_UNIT, ns, root, required=True) - - # print symbol list - node_list = root.xpath("nx:symbols", namespaces=ns) - print("**Symbols**:\n") - if len(node_list) == 0: - print(" No symbol table\n") - elif len(node_list) > 1: - raise Exception("Invalid symbol table in " % root.get("name")) - else: - printDoc(INDENTATION_UNIT, ns, node_list[0]) - for node in node_list[0].xpath("nx:symbol", namespaces=ns): - doc = getDocLine(ns, node) - print(f" **{node.get('name')}**", end="") - if doc: - print(f": {doc}", end="") - print("\n") - - # print group references - print("**Groups cited**:") - node_list = root.xpath("//nx:group", namespaces=ns) - groups = [] - for node in node_list: - g = node.get("type") - if g.startswith("NX") and g not in groups: - groups.append(g) - if len(groups) == 0: - print(" none\n") - else: - out = [(":ref:`%s`" % g) for g in groups] - txt = ", ".join(sorted(out)) - print(f" {txt}\n") - out = [("%s (base class); used in %s" % (g, listing_category)) for g in groups] - txt = ", ".join(out) - print(f".. index:: {txt}\n") - - # TODO: change instances of \t to proper indentation - - # print full tree - print("**Structure**:\n") - for subnode in root.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, "file", subnode, optional, INDENTATION_UNIT, parent_path - ) # FIXME: +"/"+name ) - printFullTree(ns, root, name, INDENTATION_UNIT, parent_path) - - printAnchorList() - - # print NXDL source location - print("") - print("**NXDL Source**:") - print(f" {HTML_ROOT}/{nxdl_subdir}/{name}.nxdl.xml") - - -def main(): - """ - standard command-line processing - """ - import argparse - - parser = argparse.ArgumentParser(description="test nxdl2rst code") - parser.add_argument("nxdl_file", help="name of NXDL file") - results = parser.parse_args() - nxdl_file = results.nxdl_file - - if not os.path.exists(nxdl_file): - print(f"Cannot find {nxdl_file}") - exit() - - print_rst_from_nxdl(nxdl_file) - - # if the NXDL has a subdirectory, - # copy that subdirectory (quietly) to the pwd, such as: - # contributed/NXcanSAS.nxdl.xml: cp -a contributed/canSAS ./ - category = os.path.basename(os.getcwd()) - path = os.path.join("../../../../", category) - basename = os.path.basename(nxdl_file) - corename = basename[2:].split(".")[0] - source = os.path.join(path, corename) - if os.path.exists(source): - target = os.path.join(".", corename) - replicate(source, target) - - -if __name__ == "__main__": - WRITE_ANCHOR_REGISTRY = True - main() - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/nxdl_desc2rst.py b/utils/nxdl_desc2rst.py deleted file mode 100755 index 5d9a49a22c..0000000000 --- a/utils/nxdl_desc2rst.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python - -""" -Read the NXDL field types specification and find -all the valid data types. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import os, sys -import lxml.etree -import textwrap - - -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 - - -ELEMENT_DICT = { - "attribute": """ -An ``attribute`` element can *only* be a child of a -``field`` or ``group`` element. -It is used to define *attribute* elements to be used and their data types -and possibly an enumeration of allowed values. - -For more details, see: -:ref:`NXDL.data.type.attributeType` - """, - "definition": """ -A ``definition`` element can *only* be used -at the root level of an NXDL specification. -Note: Due to the large number of attributes of the ``definition`` element, -they have been omitted from the figure below. - -For more details, see: -:ref:`NXDL.data.type.definition`, -:ref:`NXDL.data.type.definitionType`, and -:ref:`NXDL.data.type.definitionTypeAttr` - """, - "dimensions": """ -The ``dimensions`` element describes the *shape* of an array. -It is used *only* as a child of a ``field`` element. - -For more details, see: -:ref:`NXDL.data.type.dimensionsType` - """, - "doc": """ -A ``doc`` element can be a child of most NXDL elements. In most cases, the -content of the ``doc`` element will also become part of the NeXus manual. - -:element: {any}: - -In documentation, it may be useful to -use an element that is not directly specified by the NXDL language. -The *any* element here says that one can use any element -at all in a ``doc`` element and NXDL will not process it but pass it through. - -For more details, see: -:ref:`NXDL.data.type.docType` - """, - "enumeration": """ -An ``enumeration`` element can *only* be a child of a -``field`` or ``attribute`` element. -It is used to restrict the available choices to a predefined list, -such as to control varieties in spelling of a controversial word (such as -*metre* vs. *meter*). - -For more details, see: -:ref:`NXDL.data.type.enumerationType` - """, - "field": """ -The ``field`` element provides the value of a named item. Many different attributes -are available to further define the ``field``. Some of the attributes are not -allowed to be used together (such as ``axes`` and ``axis``); see the documentation -of each for details. -It is used *only* as a child of a ``group`` element. - -For more details, see: -:ref:`NXDL.data.type.fieldType` - """, - "choice": """ -A ``choice`` element is used when a named group might take one -of several possible NeXus base classes. Logically, it must -have at least two group children. - -For more details, see: -:ref:`NXDL.data.type.choiceType` - """, - "group": """ -A ``group`` element can *only* be a child of a -``definition`` or ``group`` element. -It describes a common level of organization in a NeXus data file, similar -to a subdirectory in a file directory tree. - -For more details, see: -:ref:`NXDL.data.type.groupType` - """, - "link": """ -.. index:: - single: link target - -A ``link`` element can *only* be a child of a -``definition``, -``field``, or ``group`` element. -It describes the path to the original source of the parent -``definition``, -``field``, or ``group``. - -For more details, see: -:ref:`NXDL.data.type.linkType` - """, - "symbols": """ -A ``symbols`` element can *only* be a child of a ``definition`` element. -It defines the array index symbols to be used when defining arrays as -``field`` elements with common dimensions and lengths. - -For more details, see: -:ref:`NXDL.data.type.symbolsType` - """, -} - -DATATYPE_DICT = { - "basicComponent": """/xs:schema//xs:complexType[@name='basicComponent']""", - "validItemName": """/xs:schema//xs:simpleType[@name='validItemName']""", - "validNXClassName": """/xs:schema//xs:simpleType[@name='validNXClassName']""", - "validTargetName": """/xs:schema//xs:simpleType[@name='validTargetName']""", - "nonNegativeUnbounded": """/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']""", -} - -ELEMENT_PREAMBLE = """ -============================= -NXDL Elements and Field Types -============================= - -The documentation in this section has been obtained directly -from the NXDL Schema file: *nxdl.xsd*. -First, the basic elements are defined in alphabetical order. -Attributes to an element are indicated immediately following the element -and are preceded with an "@" symbol, such as -**@attribute**. -Then, the common data types used within the NXDL specification are defined. -Pay particular attention to the rules for *validItemName* -and *validNXClassName*. - -.. - 2010-11-29,PRJ: - This contains a lot of special case code to lay out the NXDL chapter. - It could be cleaner but that would also involve some cooperation on - anyone who edits nxdl.xsd which is sure to break. The special case ensures - the parts come out in the chosen order. BUT, it is possible that new - items in nxdl.xsd will not automatically go in the manual. - Can this be streamlined with some common methods? - Also, there is probably too much documentation in nxdl.xsd. Obscures the function. - -.. index:: - see: attribute; NXDL attribute - ! single: NXDL elements - -.. _NXDL.elements: - -NXDL Elements -============= - - """ - -DATATYPE_PREAMBLE = """ - -.. _NXDL.data.types.internal: - -NXDL Field Types (internal) -=========================== - -Field types that define the NXDL language are described here. -These data types are defined in the XSD Schema (``nxdl.xsd``) -and are used in various parts of the Schema to define common structures -or to simplify a complicated entry. While the data types are not intended for -use in NXDL specifications, they define structures that may be used in NXDL specifications. - -""" - -DATATYPE_POSTAMBLE = """ -**The** ``xs:string`` **data type** - The ``xs:string`` data type can contain characters, - line feeds, carriage returns, and tab characters. - See https://www.w3schools.com/xml/schema_dtypes_string.asp - for more details. - -**The** ``xs:token`` **data type** - The ``xs:string`` data type is derived from the - ``xs:string`` data type. - - The ``xs:token`` data type also contains characters, - but the XML processor will remove line feeds, carriage returns, tabs, - leading and trailing spaces, and multiple spaces. - See https://www.w3schools.com/xml/schema_dtypes_string.asp - for more details. -""" - - -def _tagMatch(ns, parent, match_list): - """match this tag to a list""" - if parent is None: - raise ValueError("Must supply a valid parent node") - parent_tag = parent.tag - tag_found = False - for item in match_list: - # this routine only handles certain XML Schema components - tag_found = parent_tag == "{%s}%s" % (ns["xs"], item) - if tag_found: - break - return tag_found - - -def _indent(indentLevel): - return INDENTATION * indentLevel - - -def printTitle(title, indentLevel): - print(title) - print(TITLE_MARKERS[indentLevel] * len(title) + "\n") - - -def generalHandler(ns, parent=None, indentLevel=0): - """Handle XML nodes like the former XSLT template""" - # ignore things we don't know how to handle - known_tags = ("complexType", "simpleType", "group", "element", "attribute") - if not _tagMatch(ns, parent, known_tags): - return - - parent_name = parent.get("name") - if parent_name is None: - return - - simple_tag = parent.tag[ - parent.tag.find("}") + 1 : - ] # cut off the namespace identifier - - # ... - name = parent_name # + ' data type' - if simple_tag == "attribute": - name = "@" + name - - if indentLevel == 0 and not simple_tag in ("attribute"): - print(".. index:: ! %s (NXDL data type)\n" % name) - print("\n.. _%s:\n" % ("NXDL.data.type." + name)) - - printTitle(name, indentLevel) - - printDocs(ns, parent, indentLevel) - - if len(parent.xpath("xs:attribute", namespaces=ns)) > 0: - printTitle("Attributes of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:attribute", indentLevel + 1) - - node_list = parent.xpath("xs:restriction", namespaces=ns) - if len(node_list) > 0: - # printTitle("Restrictions of "+name, indentLevel+1) - restrictionHandler(ns, node_list[0], indentLevel + 1) - node_list = parent.xpath( - "xs:simpleType/xs:restriction/xs:enumeration", namespaces=ns - ) - if len(node_list) > 0: - # printTitle("Enumerations of "+name, indentLevel+1) - applyTemplates( - ns, - parent, - "xs:simpleType/xs:restriction", - indentLevel + 1, - handler=restrictionHandler, - ) - - if len(parent.xpath("xs:sequence/xs:element", namespaces=ns)) > 0: - printTitle("Elements of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:sequence/xs:element", indentLevel + 1) - - node_list = parent.xpath("xs:sequence/xs:group", namespaces=ns) - if len(node_list) > 0: - printTitle("Groups under " + name, indentLevel + 1) - printDocs(ns, node_list[0], indentLevel + 1) - - applyTemplates(ns, parent, "xs:simpleType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType/xs:attribute", indentLevel + 1) - applyTemplates( - ns, parent, "xs:complexContent/xs:extension/xs:attribute", indentLevel + 1 - ) - applyTemplates( - ns, parent, "xs:complexType/xs:sequence/xs:attribute", indentLevel + 1 - ) - applyTemplates(ns, parent, "xs:complexType/xs:sequence/xs:element", indentLevel + 1) - applyTemplates( - ns, - parent, - "xs:complexContent/xs:extension/xs:sequence/xs:element", - indentLevel + 1, - ) - - -def restrictionHandler(ns, parent=None, indentLevel=0): - """Handle XSD restriction nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ("restriction",)): - return - printDocs(ns, parent, indentLevel) - print("\n") - print(_indent(indentLevel) + "The value may be any") - base = parent.get("base") - pattern_nodes = parent.xpath("xs:pattern", namespaces=ns) - enumeration_nodes = parent.xpath("xs:enumeration", namespaces=ns) - if len(pattern_nodes) > 0: - print( - _indent(indentLevel) - + "``%s``" % base - + " that *also* matches the regular expression::\n" - ) - print(_indent(indentLevel) + " " * 4 + pattern_nodes[0].get("value")) - elif len(pattern_nodes) > 0: - # how will this be reached? Perhaps a deprecated procedure - print(_indent(indentLevel) + "``%s``" % base + " from this list:") - for node in enumeration_nodes: - enumerationHandler(ns, node, indentLevel) - printDocs(ns, node, indentLevel) - print(_indent(indentLevel)) - elif len(enumeration_nodes) > 0: - print(_indent(indentLevel) + "one from this list only:\n") - for node in enumeration_nodes: - enumerationHandler(ns, node, indentLevel) - printDocs(ns, parent, indentLevel) - print(_indent(indentLevel)) - else: - print("@" + base) - print("\n") - - -def enumerationHandler(ns, parent=None, indentLevel=0): - """Handle XSD enumeration nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ["enumeration"]): - return - print(_indent(indentLevel) + "* ``%s``" % parent.get("value")) - printDocs(ns, parent, indentLevel) - - -def applyTemplates(ns, parent, path, indentLevel, handler=generalHandler): - """iterate the nodes found on the supplied XPath expression""" - db = {} - for node in parent.xpath(path, namespaces=ns): - name = node.get("name") or node.get("ref") or node.get("value") - if name is not None: - if name in ("nx:groupGroup",): - print(">" * 45, name) - if name in db: - raise KeyError("Duplicate name found: " + name) - db[name] = node - for name in sorted(db): - node = db[name] - handler(ns, node, indentLevel) - # printDocs(ns, node, indentLevel) - - -def printDocs(ns, parent, indentLevel=0): - docs = getDocFromNode(ns, parent) - if docs is not None: - print(_indent(indentLevel) + "\n") - for line in docs.splitlines(): - print(_indent(indentLevel) + line) - print(_indent(indentLevel) + "\n") - - -def getDocFromNode(ns, node, retval=None): - annotation_node = node.find("xs:annotation", ns) - if annotation_node is None: - return retval - documentation_node = annotation_node.find("xs:documentation", ns) - if documentation_node is None: - return retval - - # Be sure to grab _all_ content in the node. - # In the documentation nodes, use XML entities ("<"" instead of "<") - # for documentation characters that would otherwise be considered as XML. - s = lxml.etree.tostring(documentation_node, method="text", pretty_print=True) - rst = s.decode().lstrip("\n") # remove any leading blank lines - rst = rst.rstrip() # remove any trailing white space - text = textwrap.dedent(rst) # remove common leading space - - # substitute HTML entities in markup: "<" for "<" - # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string - try: # see #661 - import html - - text = html.unescape(text) - except (ImportError, AttributeError): - from html import parser as HTMLParser - - htmlparser = HTMLParser.HTMLParser() - text = htmlparser.unescape(text) - - return text.lstrip() - - -def addFigure(name, indentLevel=0): - fmt = """ -.. compound:: - - .. _%s: - - .. figure:: %s - :alt: fig.nxdl/nxdl_%s - :width: %s - - Graphical representation of the NXDL ``%s`` element - - .. Images of NXDL structure are generated from nxdl.xsd source - using the Eclipse XML Schema Editor (Web Tools Platform). Open the nxdl.xsd file and choose the - "Design" tab. Identify the structure to be documented and double-click to expand - as needed to show the detail. Use the XSD > "Export Diagram as Image ..." menu item (also available - as button in top toolbar). - Set the name: "nxdl_%s.png" and move the file into the correct location using - your operating system's commands. Commit the revision to version control. - """ - imageFile = "img/nxdl/nxdl_%s.png" % name - figure_id = "fig.nxdl_%s" % name - if not os.path.exists(os.path.abspath(imageFile)): - return - text = fmt % (figure_id, imageFile, name, "80%", name, name,) - indent = _indent(indentLevel) - for line in text.splitlines(): - print(indent + line) - print("\n") - - -def pickNodesFromXpath(ns, parent, path): - return parent.xpath(path, namespaces=ns) - - -def main(tree, ns): - print(".. auto-generated by script: " + sys.argv[0]) - print(ELEMENT_PREAMBLE) - - for name in sorted(ELEMENT_DICT): - print("") - print(".. index:: ! %s (NXDL element)\n" % name) - print(".. _%s:\n" % name) - printTitle(name, indentLevel=0) - print("\n") - print(ELEMENT_DICT[name]) - print("\n") - addFigure(name, indentLevel=0) - - print(DATATYPE_PREAMBLE) - - path_list = ( - "/xs:schema/xs:complexType[@name='attributeType']", - "/xs:schema/xs:element[@name='definition']", - "/xs:schema/xs:complexType[@name='definitionType']", - "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", - "/xs:schema/xs:complexType[@name='dimensionsType']", - "/xs:schema/xs:complexType[@name='docType']", - "/xs:schema/xs:complexType[@name='enumerationType']", - "/xs:schema/xs:complexType[@name='fieldType']", - "/xs:schema/xs:complexType[@name='choiceType']", - "/xs:schema/xs:complexType[@name='groupType']", - "/xs:schema/xs:complexType[@name='linkType']", - "/xs:schema/xs:complexType[@name='symbolsType']", - "/xs:schema/xs:complexType[@name='basicComponent']", - "/xs:schema/xs:simpleType[@name='validItemName']", - "/xs:schema/xs:simpleType[@name='validNXClassName']", - "/xs:schema/xs:simpleType[@name='validTargetName']", - "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", - ) - for path in path_list: - nodes = pickNodesFromXpath(ns, tree, path) - print("\n.. Xpath = %s\n" % path) - generalHandler(ns, parent=nodes[0]) - - print(DATATYPE_POSTAMBLE) - - -if __name__ == "__main__": - developermode = True - developermode = False - if developermode and len(sys.argv) != 2: - path = os.path.dirname(__file__) - NXDL_SCHEMA_FILE = os.path.join(path, "..", "nxdl.xsd") - else: - if len(sys.argv) != 2: - print("usage: %s nxdl.xsd" % sys.argv[0]) - exit() - NXDL_SCHEMA_FILE = sys.argv[1] - if not os.path.exists(NXDL_SCHEMA_FILE): - print("Cannot find %s" % NXDL_SCHEMA_FILE) - exit() - - tree = lxml.etree.parse(NXDL_SCHEMA_FILE) - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - - main(tree, ns) - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/test_nxdl.py b/utils/test_nxdl.py deleted file mode 100644 index e7152c7941..0000000000 --- a/utils/test_nxdl.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/env python - -""" -unit testing of NeXus definitions NXDL files and XML Schema -""" - -import os -import sys -import unittest -import lxml.etree - -# xmllint --noout --schema nxdl.xsd base_classes/NXentry.nxdl.xml -# base_classes/NXentry.nxdl.xml validates - - -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -NXDL_XSD_SCHEMA = "nxdl.xsd" -NXDL_SCHEMA = lxml.etree.XMLSchema( - lxml.etree.parse(os.path.join(BASE_DIR, NXDL_XSD_SCHEMA)) -) - -NXDL_CATEGORY_NAMES = "base_classes applications contributed_definitions".split() - - -class NXDL_Invalid(Exception): - pass - - -class NXDL_Valid(Exception): - pass - - -def isNXDL(fname): - return fname.endswith(".nxdl.xml") - - -def get_NXDL_file_list(): - os.chdir(BASE_DIR) - file_list = [] - for category in NXDL_CATEGORY_NAMES: - raw_list = os.listdir(category) - nxdl_files = [os.path.join(category, fn) for fn in raw_list if isNXDL(fn)] - file_list += sorted(nxdl_files) - return file_list - - -def validate_xml(xml_file_name): - """ - validate an NXDL XML file against an XML Schema file - - :param str xml_file_name: name of XML file - """ - try: - xml_tree = lxml.etree.parse(xml_file_name) - except lxml.etree.XMLSyntaxError as exc: - msg = xml_file_name + " : " + str(exc) - raise NXDL_Invalid(msg) - try: - result = NXDL_SCHEMA.assertValid(xml_tree) - # there is no assertNotRaises so raise this when successful - raise NXDL_Valid - except lxml.etree.DocumentInvalid as exc: - msg = xml_file_name + " : " + str(exc) - raise NXDL_Invalid(msg) - - -class TestMaker(type): - def __new__(cls, clsname, bases, dct): - # Add a method to the class' __dict__ for every - # file name in the NXDL file list. - cat_number_dict = {c: str(i + 1) for i, c in enumerate(NXDL_CATEGORY_NAMES)} - for fname in get_NXDL_file_list(): - category, nxdl_name = os.path.split(fname) - category_number = cat_number_dict[category] - point = nxdl_name.find(".") - nxdl_name = nxdl_name[:point] - test_name = "test" - # since these will be sorted, get the categories in the desired order - test_name += "__" + str(category_number) - test_name += "__" + category - test_name += "__" + nxdl_name - dct[test_name] = cls.make_test(fname) - - return super(TestMaker, cls).__new__(cls, clsname, bases, dct) - - @staticmethod - def make_test(nxdl_file_name): - def test_wrap(self): - # test body for each NXDL file test - with self.assertRaises(NXDL_Valid): - validate_xml(nxdl_file_name) - self.assertRaises(NXDL_Valid, validate_xml, nxdl_file_name) - - return test_wrap - - -class Individual_NXDL_Tests(unittest.TestCase, metaclass=TestMaker): - """ - run all tests created in TestMaker() class, called by suite() - """ - - -def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" - test_suite = unittest.TestSuite() - test_suite.addTests(unittest.makeSuite(Individual_NXDL_Tests)) - return test_suite - - -def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) - runner.run(suite()) - - -if __name__ == "__main__": - run() diff --git a/utils/test_nxdl2rst.py b/utils/test_nxdl2rst.py deleted file mode 100644 index 805c02ae87..0000000000 --- a/utils/test_nxdl2rst.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/env python - -""" -unit testing: NXDL to RST for documentation -""" - -import os -import sys -import unittest -import lxml.etree -from io import StringIO - -import nxdl2rst - - -class Capture_stdout(list): - """ - capture all printed output (to stdout) into list - - # http://stackoverflow.com/questions/16571150/how-to-capture-stdout-output-from-a-python-function-call - """ - - def __enter__(self): - self._stdout = sys.stdout - sys.stdout = self._stringio = StringIO() - return self - - def __exit__(self, *args): - self.extend(self._stringio.getvalue().splitlines()) - del self._stringio # free up some memory - sys.stdout = self._stdout - - -class Issue_524_Clarify_Optional_or_Required(unittest.TestCase): - """ - make it obvious what is required and what is optional - - **field**: (optional or required) NX_TYPE - """ - - def test_base_class_NXentry(self): - expected_lines = """ - **definition**: (optional) :ref:`NX_CHAR ` - **DATA**: (optional) :ref:`NXdata` - **notes**: (optional) :ref:`NXnote` - **@default**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXentry", expected_lines) - - def test_base_class_NXuser(self): - expected_lines = """ - **name**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXuser", expected_lines) - - def test_application_definition_NXcanSAS(self): - expected_lines = """ - **definition**: (required) :ref:`NX_CHAR ` - **title**: (required) :ref:`NX_CHAR ` - **run**: (required) :ref:`NX_CHAR ` - **I**: (required) :ref:`NX_NUMBER ` - **Q**: (required) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **Idev**: (optional) :ref:`NX_NUMBER ` - **dQw**: (optional) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **dQl**: (optional) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **ENTRY**: (required) :ref:`NXentry` - **DATA**: (required) :ref:`NXdata` - **TRANSMISSION_SPECTRUM**: (optional) :ref:`NXdata` - **SAMPLE**: (optional) :ref:`NXsample` - **INSTRUMENT**: (optional) :ref:`NXinstrument` - **NOTE**: (optional) :ref:`NXnote` - **PROCESS**: (optional) :ref:`NXprocess` - **SOURCE**: (optional) :ref:`NXsource` - **@default**: (optional) :ref:`NX_CHAR ` - **@timestamp**: (optional) :ref:`NX_DATE_TIME ` - **@canSAS_class**: (required) :ref:`NX_CHAR ` - **@signal**: (required) :ref:`NX_CHAR ` - **@I_axes**: (required) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("applications", "NXcanSAS", expected_lines) - - def apply_tests(self, category, class_name, expected_lines): - nxdl_file = os.path.join( - os.path.dirname(__file__), "..", category, class_name + ".nxdl.xml" - ) - self.assertTrue(os.path.exists(nxdl_file), nxdl_file) - - sys.argv.insert(0, "python") - with Capture_stdout() as printed_lines: - nxdl2rst.print_rst_from_nxdl(nxdl_file) - - printed_lines = [_.strip() for _ in printed_lines] - for line in expected_lines: - expected = line.strip() - self.assertTrue(expected in printed_lines, line.strip()) - - -def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" - test_suite = unittest.TestSuite() - test_suite_list = [ - Issue_524_Clarify_Optional_or_Required, - ] - for item in test_suite_list: - test_suite.addTests(unittest.makeSuite(item)) - return test_suite - - -def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) - runner.run(suite()) - - -if __name__ == "__main__": - run() diff --git a/utils/test_suite.py b/utils/test_suite.py deleted file mode 100644 index 92b8bb246b..0000000000 --- a/utils/test_suite.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -""" -unit testing of the NeXus definitions -""" - -import os -import unittest -import sys - - -def suite(*args, **kw): - import test_nxdl - import test_nxdl2rst - - test_suite = unittest.TestSuite() - test_list = [ - test_nxdl, - test_nxdl2rst, - ] - - for test in test_list: - test_suite.addTest(test.suite()) - return test_suite - - -if __name__ == "__main__": - owd = os.getcwd() - runner = unittest.TextTestRunner(verbosity=2) - result = runner.run(suite()) - os.chdir(owd) - sys.exit(len(result.errors)) diff --git a/utils/types2rst.py b/utils/types2rst.py deleted file mode 100755 index 2faa1a62fc..0000000000 --- a/utils/types2rst.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -""" -Read the the NeXus NXDL types specification and find -all the valid data types. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import units2rst - - -if __name__ == "__main__": - units2rst.worker("primitiveType", section="data") - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/units2rst.py b/utils/units2rst.py deleted file mode 100755 index ad10ddfeb4..0000000000 --- a/utils/units2rst.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -""" -Read the the NeXus NXDL types specification and find -all the valid types of units. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import os, sys -import lxml.etree - - -def worker(nodeMatchString, section="units"): - if len(sys.argv) != 2: - print("usage: %s nxdlTypes.xsd" % sys.argv[0]) - exit() - NXDL_TYPES_FILE = sys.argv[1] - if not os.path.exists(NXDL_TYPES_FILE): - print("Cannot find %s" % NXDL_TYPES_FILE) - exit() - - tree = lxml.etree.parse(NXDL_TYPES_FILE) - - output = [".. auto-generated by %s -- DO NOT EDIT" % sys.argv[0]] - output.append("") - - labels = ("term", "description") - output.append(".. nodeMatchString : %s" % nodeMatchString) - output.append("") - db = {} - - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - root = tree.xpath("//xs:schema", namespaces=ns)[0] - s = "//xs:simpleType" - node_list = tree.xpath("//xs:simpleType", namespaces=ns) - - # get the names of all the types of units - members = [] - for node in node_list: - if node.get("name") == nodeMatchString: - union = node.xpath("xs:union", namespaces=ns) - members = union[0].get("memberTypes", "").split() - - # get the definition of each type of units - for node in node_list: - node_name = node.get("name") - if node_name is None: - continue - if "nxdl:" + node_name in members: - words = node.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] - examples = [] - for example in words.iterchildren(): - nm = example.attrib.get("name") - if nm is not None and nm == "example": - examples.append("``" + example.text + "``") - a = words.text - if len(examples) > 0: - a = " ".join(a.split()) + ",\n\texample(s): " + " | ".join(examples) - db[node_name] = a - - # for item in node.xpath('xs:restriction//xs:enumeration', namespaces=ns): - # key = '%s' % item.get('value') - # words = item.xpath('xs:annotation/xs:documentation', namespaces=ns)[0] - # db[key] = words.text - - print("\n".join(output)) - - # this list is too long to make this a table in latex - # for two columns, a Sphinx fieldlist will do just as well - for key in sorted(db): - print(".. index:: ! %s (%s type)\n" % (key, section)) # index entry - print(".. _%s:\n" % key) # cross-reference point - print(":%s:" % key) - for line in db[key].splitlines(): - print(" %s" % line) - print("") - - -if __name__ == "__main__": - # sys.argv.append('../nxdlTypes.xsd') # FIXME: developer only -- remove for production!!! - worker("anyUnitsAttr") - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/update_copyright_date.py b/utils/update_copyright_date.py index af7b73fca1..261b1933a3 100755 --- a/utils/update_copyright_date.py +++ b/utils/update_copyright_date.py @@ -12,7 +12,6 @@ import os, sys import mimetypes -import local_utilities from build_preparation import ROOT_DIR_EXPECTED_RESOURCES import datetime