From 2c570831a4ebe1b77a507809e72612527e471e6b Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Sun, 26 Jun 2022 12:09:26 +0200 Subject: [PATCH 1/6] NeXus developer tools to replace the util scripts --- .github/workflows/ci.yaml | 98 +++++ README.md | 3 +- dev_tools/README.md | 102 +++++ dev_tools/__init__.py | 0 dev_tools/__main__.py | 53 +++ dev_tools/apps/__init__.py | 0 dev_tools/apps/dir_app.py | 31 ++ dev_tools/apps/impatient_app.py | 15 + dev_tools/apps/manual_app.py | 138 +++++++ dev_tools/apps/nxclass_app.py | 109 ++++++ dev_tools/docs/__init__.py | 4 + dev_tools/docs/anchor_list.py | 217 +++++++++++ dev_tools/docs/nxdl.py | 640 +++++++++++++++++++++++++++++++ dev_tools/docs/nxdl_index.py | 145 +++++++ dev_tools/docs/xsd.py | 503 ++++++++++++++++++++++++ dev_tools/docs/xsd_units.py | 72 ++++ dev_tools/globals/__init__.py | 0 dev_tools/globals/directories.py | 115 ++++++ dev_tools/globals/errors.py | 14 + dev_tools/globals/nxdl.py | 21 + dev_tools/globals/urls.py | 2 + dev_tools/nxdl/__init__.py | 10 + dev_tools/nxdl/discover.py | 30 ++ dev_tools/nxdl/syntax.py | 28 ++ dev_tools/requirements.txt | 14 + dev_tools/setup.cfg | 8 + dev_tools/tests/__init__.py | 0 dev_tools/tests/test_docs.py | 52 +++ dev_tools/tests/test_nxdl.py | 30 ++ dev_tools/utils/__init__.py | 0 dev_tools/utils/copy.py | 50 +++ dev_tools/utils/diff.py | 28 ++ dev_tools/utils/types.py | 4 + 33 files changed, 2534 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ci.yaml create mode 100644 dev_tools/README.md create mode 100644 dev_tools/__init__.py create mode 100644 dev_tools/__main__.py create mode 100644 dev_tools/apps/__init__.py create mode 100644 dev_tools/apps/dir_app.py create mode 100644 dev_tools/apps/impatient_app.py create mode 100644 dev_tools/apps/manual_app.py create mode 100644 dev_tools/apps/nxclass_app.py create mode 100644 dev_tools/docs/__init__.py create mode 100644 dev_tools/docs/anchor_list.py create mode 100644 dev_tools/docs/nxdl.py create mode 100644 dev_tools/docs/nxdl_index.py create mode 100644 dev_tools/docs/xsd.py create mode 100644 dev_tools/docs/xsd_units.py create mode 100644 dev_tools/globals/__init__.py create mode 100644 dev_tools/globals/directories.py create mode 100644 dev_tools/globals/errors.py create mode 100644 dev_tools/globals/nxdl.py create mode 100644 dev_tools/globals/urls.py create mode 100644 dev_tools/nxdl/__init__.py create mode 100644 dev_tools/nxdl/discover.py create mode 100644 dev_tools/nxdl/syntax.py create mode 100644 dev_tools/requirements.txt create mode 100644 dev_tools/setup.cfg create mode 100644 dev_tools/tests/__init__.py create mode 100644 dev_tools/tests/test_docs.py create mode 100644 dev_tools/tests/test_nxdl.py create mode 100644 dev_tools/utils/__init__.py create mode 100644 dev_tools/utils/copy.py create mode 100644 dev_tools/utils/diff.py create mode 100644 dev_tools/utils/types.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000000..357145fc13 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,98 @@ +name: CI + +on: + push: + branches: + - main # push commit to the main branch + tags: + - 'v2*' # push tag starting with "v2" to the main branch + pull_request: + branches: + - main # pull request to the main branch + workflow_dispatch: # allow manual triggering + +defaults: + run: + shell: bash + +jobs: + build-linux: + name: CI py${{ matrix.python-version }} + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7', '3.8', '3.9', '3.10'] + max-parallel: 5 + env: + python_version: ${{ matrix.python-version }} + + steps: + - name: Checkout Repository + uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Deploy Information + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + run: | + echo "The HTML NeXus User Manual will be pushed to" + echo " https://github.com/nexusformat/definitions/tree/gh-pages" + echo "The HTML NeXus User Manual will be deployed on" + echo " https://nexusformat.github.io/definitions/" + + - name: Install Requirements + run: | + python3 -m pip install --upgrade pip setuptools + python3 -m pip install -r dev_tools/requirements.txt + python3 -m pip list + + - name: Check Code Style + run: | + cd dev_tools + black --check . + flake8 . + isort --check . + + - name: Run Tests + run: | + pytest dev_tools -v + + - name: Install LaTeX + run: | + sudo apt-get update -y && \ + sudo apt-get install -y \ + latexmk \ + texlive-latex-recommended \ + texlive-latex-extra \ + texlive-fonts-recommended + + - name: Generate build files + run: | + python3 -m dev_tools manual --prepare + python3 -m dev_tools impatient --prepare + + - name: Build PDF Manuals + run: | + sphinx-build -M latexpdf build/impatient-guide/ build/impatient-guide/build + cp build/impatient-guide/build/latex/NXImpatient.pdf build/manual/source/_static/NXImpatient.pdf + sphinx-build -M latexpdf build/manual/source/ build/manual/build + cp build/manual/build/latex/nexus.pdf build/manual/source/_static/NeXusManual.pdf + + - name: Build HTML Manuals + run: | + sphinx-build -b html build/manual/source/ build/manual/build/html + sphinx-build -b html -W build/impatient-guide/ build/impatient-guide/build/html + + - name: Build and Commit the NeXus User Manual + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + uses: sphinx-notes/pages@master + with: + # path to the conf.py directory + documentation_path: build/manual/source + + - name: Deploy the NeXus User Manual + if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + branch: gh-pages diff --git a/README.md b/README.md index bb79e5b1f5..2758ea621c 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,7 @@ * Documentation: https://manual.nexusformat.org/ * Release Notes: https://github.com/nexusformat/definitions/wiki/Release-Notes * License: [![License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) -* Continuous Integration: [![Syntax Checking](https://github.com/nexusformat/definitions/actions/workflows/syntax-checks.yml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/syntax-checks.yml) -* Continuous Deployment: [![Publish Documentation](https://github.com/nexusformat/definitions/actions/workflows/publish-sphinx.yml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/publish-sphinx.yml) +* Test, Build and Deploy: [![Workflow](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml) These are the components that define the structure of NeXus data files in the development directory. diff --git a/dev_tools/README.md b/dev_tools/README.md new file mode 100644 index 0000000000..4f42b489e1 --- /dev/null +++ b/dev_tools/README.md @@ -0,0 +1,102 @@ +# NeXus development tools + +The NeXus development tools are used to validate NeXus class definitions +and generate all files needed for documentation building with sphinx. + +## Quick reference + +Install all requirements + +```bash +python3 -m pip install -r dev_tools/requirements.txt +``` + +Run all tests + +```bash +pytest dev_tools +``` + +Generate the build files + +```bash +python3 -m dev_tools impatient --prepare +python3 -m dev_tools manual --prepare +``` + +Build the PDF manuals + +```bash +sphinx-build -M latexpdf build/impatient-guide/ build/impatient-guide/build +cp build/impatient-guide/build/latex/NXImpatient.pdf build/manual/source/_static/NXImpatient.pdf + +sphinx-build -M latexpdf build/manual/source/ build/manual/build +cp build/manual/build/latex/nexus.pdf build/manual/source/_static/NeXusManual.pdf +``` + +Build the HTML manuals + +```bash +sphinx-build -b html -W build/impatient-guide/ build/impatient-guide/build/html +sphinx-build -b html -W build/manual/source/ build/manual/build/html +``` + +Auto-formatting and syntax checking to the developer tools themselves + +```bash +cd dev_tools +black . +isort . +flake8 . +``` + +## Prepare environment (optional) + +Create a fresh python environment + +```bash +python3 -m venv nexusenv +``` + +Activate the environment on Linux or MacOS + +```bash +source nexusenv/bin/activate +``` + +Activate the environment on Windows + +```bash +nexusenv\Scripts\activate.bat +``` + +## Command line interface + +```bash +python3 -m dev_tools --help +``` + +### Single NeXus class definition + +This subcommand provides syntax checking and documentation building +(print, save and diff) for a single NeXus class definition + +```bash +python3 -m dev_tools nxclass --help +``` + +For example the difference between the existing documentation of +a NeXus class definition and the generated documentation + +```bash +python3 -m dev_tools nxclass nxaperture --diff +--- build + ++++ source + +@@ -1,4 +1,4 @@ + +-.. auto-generated by script ../../../../utils/nxdl2rst.py from the NXDL source NXaperture.nxdl.xml + ++.. auto-generated by dev_tools.docs.rst from the NXDL source base_classes/NXaperture.nxdl.xml +``` diff --git a/dev_tools/__init__.py b/dev_tools/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/__main__.py b/dev_tools/__main__.py new file mode 100644 index 0000000000..6e728a8e0a --- /dev/null +++ b/dev_tools/__main__.py @@ -0,0 +1,53 @@ +import argparse +import sys + +from .apps import dir_app +from .apps import impatient_app +from .apps import manual_app +from .apps import nxclass_app + + +def main(argv=None): + parser = argparse.ArgumentParser(description="NeXus development tools") + + subparsers = parser.add_subparsers(help="Commands", dest="command") + + nxclass_parser = subparsers.add_parser( + "nxclass", help="Test and documentation for a single NeXus class" + ) + nxclass_app.nxclass_args(nxclass_parser) + dir_app.dir_args(nxclass_parser) + + manual_parser = subparsers.add_parser( + "manual", help="Test and prepare User Manual building" + ) + manual_app.manual_args(manual_parser) + dir_app.dir_args(manual_parser) + + impatient_parser = subparsers.add_parser( + "impatient", help="Prepare Impatient Guide building" + ) + impatient_app.impatient_args(impatient_parser) + dir_app.dir_args(impatient_parser) + + if argv is None: + argv = sys.argv + args = parser.parse_args(argv[1:]) + + if args.command == "nxclass": + dir_app.dir_exec(args) + nxclass_app.nxclass_exec(args) + elif args.command == "manual": + dir_app.dir_exec(args) + manual_app.manual_exec(args) + elif args.command == "impatient": + dir_app.dir_exec(args) + impatient_app.impatient_exec(args) + else: + parser.print_help() + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/dev_tools/apps/__init__.py b/dev_tools/apps/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/apps/dir_app.py b/dev_tools/apps/dir_app.py new file mode 100644 index 0000000000..ddc04b10e9 --- /dev/null +++ b/dev_tools/apps/dir_app.py @@ -0,0 +1,31 @@ +from ..globals import directories + + +def dir_args(parser): + parser.add_argument( + "--source-root", + type=str, + default=None, + help="Source root directory" f" Default: {directories.get_source_root()}", + ) + parser.add_argument( + "--nxdl-root", + type=str, + default=None, + help="NXDL root directory" f" Default: {directories.get_nxdl_root()}", + ) + parser.add_argument( + "--build-root", + type=str, + default=None, + help="Build root directory" f" Default: {directories.get_build_root()}", + ) + + +def dir_exec(args): + if args.source_root: + directories.set_source_root(args.source_root) + if args.nxdl_root: + directories.set_nxdl_root(args.nxdl_root) + if args.build_root: + directories.set_build_root(args.build_root) diff --git a/dev_tools/apps/impatient_app.py b/dev_tools/apps/impatient_app.py new file mode 100644 index 0000000000..b915435820 --- /dev/null +++ b/dev_tools/apps/impatient_app.py @@ -0,0 +1,15 @@ +from ..globals import directories +from ..utils.copy import copydir + + +def impatient_args(parser): + parser.add_argument( + "--prepare", + action="store_true", + help="Create the build files for the NeXus Impatient Guide", + ) + + +def impatient_exec(args): + if args.prepare: + copydir(directories.impatient_source_root(), directories.impatient_build_root()) diff --git a/dev_tools/apps/manual_app.py b/dev_tools/apps/manual_app.py new file mode 100644 index 0000000000..a03dbf7bb3 --- /dev/null +++ b/dev_tools/apps/manual_app.py @@ -0,0 +1,138 @@ +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..docs import XSDDocGenerator +from ..docs.nxdl_index import nxdl_indices +from ..docs.xsd_units import generate_xsd_units_doc +from ..globals import directories +from ..nxdl import iter_definitions +from ..nxdl import validate_definition +from ..utils.copy import copy_files +from ..utils.copy import copydir +from ..utils.copy import download_files +from ..utils.diff import diff_ascii +from .nxclass_app import diff_nxclass_docs +from .nxclass_app import save_nxclass_docs + + +def manual_args(parser): + parser.add_argument( + "--test", + action="store_true", + help="Validate all NeXus class definitions", + ) + parser.add_argument( + "--prepare", + action="store_true", + help="Create the build files for the NeXus Impatient Guide", + ) + parser.add_argument( + "--diff", + action="store_true", + help="Print all changes in the generated documentation", + ) + + +def manual_exec(args): + # Copy the documentation source files to the build directory + if args.prepare: + copydir(directories.manual_source_root(), directories.manual_build_root()) + + # XSD and NXDL document generators + generate_docs = args.prepare or args.diff + if generate_docs: + generator = NXClassDocGenerator() + xsdgenerator = XSDDocGenerator() + if args.prepare: + output_path = directories.manual_build_staticroot() + else: + output_path = None + anchor_registry = AnchorRegistry(output_path=output_path) + + # Generate the NeXus class documentation files in the build directory + for subdir in ("base_classes", "applications", "contributed_definitions"): + for nxdl_file in iter_definitions(subdir): + if args.test: + validate_definition(nxdl_file) + if generate_docs: + rst_lines = generator(nxdl_file, anchor_registry=anchor_registry) + if args.diff: + diff_nxclass_docs(nxdl_file, rst_lines) + if args.prepare: + save_nxclass_docs(nxdl_file, rst_lines) + + # Generate the NXDL XSD documentation in the build directory + if generate_docs: + xsd_file = directories.get_xsd_file() + rst_lines = xsdgenerator(xsd_file) + nxdl_desc = directories.manual_build_sphinxsroot() / "nxdl_desc.rst" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + if generate_docs: + xsd_file = directories.get_xsd_units_file() + rst_lines = generate_xsd_units_doc(xsd_file, "anyUnitsAttr", "units") + nxdl_desc = directories.manual_build_sphinxsroot() / "units.table" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation (units)", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + if generate_docs: + xsd_file = directories.get_xsd_units_file() + rst_lines = generate_xsd_units_doc(xsd_file, "primitiveType", "data") + nxdl_desc = directories.manual_build_sphinxsroot() / "types.table" + if args.diff: + diff_ascii(xsd_file, rst_lines, nxdl_desc) + if args.prepare: + print("generate XSD documentation (types)", nxdl_desc) + with open(nxdl_desc, "w") as fh: + fh.writelines(rst_lines) + + # Generate the NeXus class documentation index files in the + # build directory so the files generated above are included in the docs. + if generate_docs: + for name, adict in nxdl_indices().items(): + index_file = adict["index_file"] + rst_lines = adict["rst_lines"] + if args.diff: + diff_ascii(name, rst_lines, index_file) + if args.prepare: + print("generate NXDL index", index_file) + if args.prepare: + with open(index_file, "w") as fh: + fh.writelines(rst_lines) + + # Generate the anchor list in several format + if args.prepare: + print("generate anchor list files in", output_path) + anchor_registry.write() + copy_files(EXTRA_FILES) + download_files(EXTRA_URLS) + + +# Path relative to source directory, +# Path relative to build directory, +# Overwrite (boolean) +EXTRA_FILES = [["NXDL_VERSION", "NXDL_VERSION", True], ["LGPL.txt", "LGPL.txt", True]] + +# URL, +# Path relative to build directory, +# Overwrite (boolean) +EXTRA_URLS = [ + [ + "https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf", + "manual/source/_static/NeXusIntern.pdf", + False, + ], + [ + "https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf", + "manual/source/_static/NXtranslate.pdf", + False, + ], +] diff --git a/dev_tools/apps/nxclass_app.py b/dev_tools/apps/nxclass_app.py new file mode 100644 index 0000000000..9023108d69 --- /dev/null +++ b/dev_tools/apps/nxclass_app.py @@ -0,0 +1,109 @@ +import shutil +from pathlib import Path +from typing import List + +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..globals import directories +from ..nxdl import find_definition +from ..nxdl import validate_definition +from ..utils.diff import diff_ascii + + +def nxclass_args(parser): + parser.add_argument( + "name", + type=str, + help="NeXus class name (For example 'nxdata')", + ) + parser.add_argument( + "--test", + action="store_true", + help="Validate the NeXus class definition", + ) + parser.add_argument( + "--prepare", + action="store_true", + help="Save the NeXus class documentation in the build directory", + ) + parser.add_argument( + "--diff", + action="store_true", + help="Print changes in the NeXus class documentation", + ) + parser.add_argument( + "--print", + action="store_true", + help="Print the NeXus class documentation", + ) + + +def nxclass_exec(args): + nxdl_file = find_definition(args.name) + assert nxdl_file, f"No definition found for {args.name}" + + if args.test: + validate_definition(nxdl_file) + + if args.prepare or args.diff or args.print: + generator = NXClassDocGenerator() + if args.prepare: + output_path = directories.manual_build_staticroot() + else: + output_path = None + anchor_registry = AnchorRegistry(output_path=output_path) + rst_lines = generator(nxdl_file, anchor_registry=anchor_registry) + + if args.prepare: + save_nxclass_docs(nxdl_file, rst_lines) + print("add to anchor list files in", output_path) + anchor_registry.write() + + if args.diff: + diff_nxclass_docs(nxdl_file, rst_lines) + + if args.print: + print("".join(rst_lines)) + + +def save_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: + """Build the NXDL file: this means prepare the documentation + and resources in the build directory. + """ + rst_file_name = get_rst_filename(nxdl_file) + + # Save the documentation in the build directory + print("generate", nxdl_file) + print(" ->", rst_file_name) + rst_file_name.parent.mkdir(parents=True, exist_ok=True) + with open(rst_file_name, "w") as fh: + fh.writelines(rst_lines) + + # Copy resources to the build directory + resource_name = rst_file_name.stem[2:] # (e.g. NXaperture -> aperture) + nxdl_resource_dir = nxdl_file.parent / resource_name + if nxdl_resource_dir.is_dir(): + rst_resource_dir = rst_file_name.parent / resource_name + print("copy", nxdl_resource_dir) + print(" ->", rst_resource_dir) + shutil.copytree(nxdl_resource_dir, rst_resource_dir, dirs_exist_ok=True) + + # Copy the NXDL file as it might be used in a "literalinclude" + nxdl_file2 = rst_file_name.parent / nxdl_file.name + print("generate", nxdl_file) + print(" ->", nxdl_file2) + shutil.copyfile(nxdl_file, nxdl_file2) + + +def diff_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: + """Build the NXDL file: this means prepare the documentation + and resources in the build directory. + """ + rst_file_name = get_rst_filename(nxdl_file) + diff_ascii(nxdl_file, rst_lines, rst_file_name) + + +def get_rst_filename(nxdl_file: Path) -> Path: + rst_file_name = directories.nxclass_build_root(nxdl_file) + rst_file_name /= nxdl_file.with_suffix("").with_suffix(".rst").name + return rst_file_name diff --git a/dev_tools/docs/__init__.py b/dev_tools/docs/__init__.py new file mode 100644 index 0000000000..a0fcda82c7 --- /dev/null +++ b/dev_tools/docs/__init__.py @@ -0,0 +1,4 @@ +from .anchor_list import AnchorRegistry # noqa F401 +from .nxdl import NXClassDocGenerator # noqa F401 +from .nxdl_index import nxdl_indices # noqa F401 +from .xsd import XSDDocGenerator # noqa F401 diff --git a/dev_tools/docs/anchor_list.py b/dev_tools/docs/anchor_list.py new file mode 100644 index 0000000000..874958cec1 --- /dev/null +++ b/dev_tools/docs/anchor_list.py @@ -0,0 +1,217 @@ +import datetime +import json +from pathlib import Path +from typing import Optional + +import lxml +import yaml + +from ..globals.nxdl import get_nxdl_version +from ..globals.urls import MANUAL_URL +from ..utils.types import PathLike + + +class AnchorRegistry: + """Document the NXDL vocabulary. Usage goes as follows + + .. code:: python + + reg = AnchorRegistry(output_path=...) + # It loaded the existing registry from file (if provided) + + reg.add(...) + reg.add(...) + ... + anchors = reg.flush_anchor_buffer() + + reg.add(...) + reg.add(...) + ... + anchors = reg.flush_anchor_buffer() + + ... + reg.write() + # It saved the in-memory registry to file (if provided) + """ + + def __init__(self, output_path: Optional[PathLike] = None) -> None: + # For example: output_path = get_build_root() / "manual" / "source" / "_static" + self._writing_enabled = bool(output_path) + if output_path: + base = "nxdl_vocabulary" + output_path = Path(output_path).absolute() + output_path.mkdir(parents=True, exist_ok=True) + self._html_file = output_path / f"{base}.html" + self._txt_file = output_path / f"{base}.txt" + self._json_file = output_path / f"{base}.json" + self._yaml_file = output_path / f"{base}.yml" + else: + self._html_file = None + self._txt_file = None + self._json_file = None + self._yaml_file = None + self._registry = self._load_registry() + self._anchor_buffer = [] + self._nxdl_file = None + self.category = None + + @property + def all_anchors(self): + result = [] + for v in self._registry.values(): + result += list(v.keys()) + return result + + @property + def nxdl_file(self) -> Optional[Path]: + return self._nxdl_file + + @nxdl_file.setter + def nxdl_file(self, value: PathLike) -> None: + self._nxdl_file = Path(value).absolute() + + def add(self, anchor): + """Add anchor to the in-memory registry and to the + current anchor buffer.""" + if anchor not in self._anchor_buffer: + self._anchor_buffer.append(anchor) + + key = self._key_from_anchor(anchor) + + if key not in self._registry: + self._registry[key] = {} + + reg = self._registry[key] + if anchor not in reg: + hanchor = self._html_anchor(anchor) + fnxdl = "/".join(self.nxdl_file.parts[-2:]).split(".")[0] + url = f"{MANUAL_URL}classes/{self.category}/{fnxdl}.html{hanchor}" + reg[anchor] = dict( + term=anchor, + html=hanchor, + url=url, + ) + + def _key_from_anchor(self, anchor): + key = anchor.lower().split("/")[-1].split("@")[-1].split("-")[0] + if "@" in anchor: + # restore preceding "@" symbol + key = "@" + key + return key + + def write(self): + """Write the in-memory registry to files""" + if not self._writing_enabled: + return + contents = dict( + _metadata=dict( + datetime=datetime.datetime.utcnow().isoformat(), + title="NeXus NXDL vocabulary.", + subtitle="Anchors for all NeXus fields, groups, " + "attributes, and links.", + version=get_nxdl_version(), + ), + terms=self._registry, + ) + self._write_yaml(contents) + self._write_json(contents) + self._write_txt() + self._write_html(contents) + + def flush_anchor_buffer(self) -> list: + """Flush the anchor buffer""" + self._anchor_buffer, ret = list(), self._anchor_buffer + return ret + + def _html_anchor(self, anchor): + """ + Create (internal hyperlink target for) HTML anchor from reST anchor. + + Example: + + * reST anchor: /NXcanSAS/ENTRY/TRANSMISSION_SPECTRUM@timestamp-attribute + * HTML anchor: #nxcansas-entry-transmission-spectrum-timestamp-attribute + """ + html_anchor = ( + anchor.lower() + .lstrip("/") + .replace("_", "-") + .replace("@", "-") + .replace("/", "-") + ) + return f"#{html_anchor}" + + def _load_registry(self) -> dict: + """Load the anchor registry in memory.""" + if not self._yaml_file: + return {} + registry = None + if self._yaml_file.exists(): + contents = yaml.load(open(self._yaml_file, "r").read(), Loader=yaml.Loader) + if contents is not None: + registry = contents.get("terms") + return registry or {} + + def _write_html(self, contents): + """Write the anchors to an HTML file.""" + if not self._html_file: + return + root = lxml.etree.Element("html") + body = lxml.etree.SubElement(root, "body") + title = lxml.etree.SubElement(body, "h1") + subtitle = lxml.etree.SubElement(body, "em") + + title.text = contents["_metadata"]["title"].strip(".") + subtitle.text = contents["_metadata"]["subtitle"].strip(".") + vocab_list = lxml.etree.SubElement(body, "h2") + vocab_list.text = "NXDL Vocabulary" + + p = lxml.etree.SubElement(body, "p") + p.text = "This content is also available in these formats: " + for ext in "json txt yml".split(): + a = lxml.etree.SubElement(p, "a") + a.attrib["href"] = f"{MANUAL_URL}_static/{self._txt_file.stem}.{ext}" + a.text = f" {ext}" + + dl = lxml.etree.SubElement(body, "dl") + for term, termlist in sorted(contents["terms"].items()): + dterm = lxml.etree.SubElement(dl, "dt") + dterm.text = term + for _, itemdict in sorted(termlist.items()): + ddef = lxml.etree.SubElement(dterm, "dd") + a = lxml.etree.SubElement(ddef, "a") + a.attrib["href"] = itemdict["url"] + a.text = itemdict["term"] + + lxml.etree.SubElement(body, "hr") + + foot = lxml.etree.SubElement(body, "p") + foot_em = lxml.etree.SubElement(foot, "em") + foot_em.text = f"written: {contents['_metadata']['datetime']}" + + html = lxml.etree.tostring(root, pretty_print=True).decode() + with open(self._html_file, "w") as f: + f.write(html) + f.write("\n") + + def _write_json(self, contents): + if not self._json_file: + return + with open(self._json_file, "w") as f: + json.dump(contents, f, indent=4) + f.write("\n") + + def _write_txt(self): + """Compendium (dump the list of all known anchors in raw form).""" + if not self._txt_file: + return + terms = self.all_anchors + with open(self._txt_file, "w") as f: + f.write("\n".join(sorted(terms))) + f.write("\n") + + def _write_yaml(self, contents): + if not self._yaml_file: + return + with open(self._yaml_file, "w") as f: + yaml.dump(contents, f) diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py new file mode 100644 index 0000000000..0c803482b0 --- /dev/null +++ b/dev_tools/docs/nxdl.py @@ -0,0 +1,640 @@ +import os +import re +from collections import OrderedDict +from html import parser as HTMLParser +from pathlib import Path +from typing import List +from typing import Optional + +import lxml + +from ..globals.directories import get_nxdl_root +from ..globals.errors import NXDLParseError +from ..globals.nxdl import NXDL_NAMESPACE +from ..globals.urls import REPO_URL +from ..utils.types import PathLike +from .anchor_list import AnchorRegistry + + +class NXClassDocGenerator: + """Generate documentation in reStructuredText markup + for a NeXus class definition.""" + + _INDENTATION_UNIT = " " * 2 + + _CATEGORY_TO_DIRNAME = { + "base": "base_classes", + "application": "applications", + } + + _CATEGORY_TO_LISTING = { + "base": "base class", + "application": "application definition", + } + + def __init__(self) -> None: + self._rst_lines = None + self._reset() + + def _reset(self): + self._anchor_registry = None + self._listing_category = None + self._use_application_defaults = None + + def __call__( + self, nxdl_file: PathLike, anchor_registry: Optional[AnchorRegistry] = None + ) -> List[str]: + self._rst_lines = list() + self._anchor_registry = anchor_registry + nxdl_file = Path(nxdl_file) + if anchor_registry: + self._anchor_registry.nxdl_file = nxdl_file + try: + try: + self._parse_nxdl_file(nxdl_file) + except Exception: + raise NXDLParseError(nxdl_file) + finally: + self._reset() + return self._rst_lines + + def _parse_nxdl_file(self, nxdl_file: Path): + assert nxdl_file.is_file() + tree = lxml.etree.parse(str(nxdl_file)) + root = tree.getroot() + + # NXDL_NAMESPACE needs to be a globally unique identifier of + # the NXDL schema. It needs to match the xmlns attribute + # in the NXDL definition of the NeXus class. + ns = {"nx": NXDL_NAMESPACE} + + nxclass_name = root.get("name") + category = root.attrib["category"] + title = nxclass_name + parent_path = "/" + nxclass_name # absolute path of parent nodes, no trailing / + if len(nxclass_name) < 2 or nxclass_name[0:2] != "NX": + raise Exception( + f'Unexpected class name "{nxclass_name}"; does not start with NX' + ) + lexical_name = nxclass_name[2:] # without padding 'NX', for indexing + + # Pass these terms to construct the full URL + if self._anchor_registry: + self._anchor_registry.category = self._CATEGORY_TO_DIRNAME[category] + + self._listing_category = self._CATEGORY_TO_LISTING[category] + self._use_application_defaults = category == "application" + + # print ReST comments and section header + source = os.path.relpath(nxdl_file, get_nxdl_root()) + self._print( + f".. auto-generated by {__name__} from the NXDL source {source} -- DO NOT EDIT" + ) + self._print("") + self._print(".. index::") + self._print(f" ! {nxclass_name} ({self._listing_category})") + self._print(f" ! {lexical_name} ({self._listing_category})") + self._print( + f" see: {lexical_name} ({self._listing_category}); {nxclass_name}" + ) + self._print("") + self._print(f".. _{nxclass_name}:\n") + self._print("=" * len(title)) + self._print(title) + self._print("=" * len(title)) + + # print category & parent class + extends = root.get("extends") + if extends is None: + extends = "none" + else: + extends = f":ref:`{extends}`" + + self._print("") + self._print("**Status**:\n") + self._print(f" {self._listing_category.strip()}, extends {extends}") + + self._print_if_deprecated(ns, root, "") + + # print official description of this class + self._print("") + self._print("**Description**:\n") + self._print_doc(self._INDENTATION_UNIT, ns, root, required=True) + + # print symbol list + node_list = root.xpath("nx:symbols", namespaces=ns) + self._print("**Symbols**:\n") + if len(node_list) == 0: + self._print(" No symbol table\n") + elif len(node_list) > 1: + raise Exception(f"Invalid symbol table in {nxclass_name}") + else: + self._print_doc(self._INDENTATION_UNIT, ns, node_list[0]) + for node in node_list[0].xpath("nx:symbol", namespaces=ns): + doc = self._get_doc_line(ns, node) + self._print(f" **{node.get('name')}**", end="") + if doc: + self._print(f": {doc}", end="") + self._print("\n") + + # print group references + self._print("**Groups cited**:") + node_list = root.xpath("//nx:group", namespaces=ns) + groups = [] + for node in node_list: + g = node.get("type") + if g.startswith("NX") and g not in groups: + groups.append(g) + if len(groups) == 0: + self._print(" none\n") + else: + out = [(f":ref:`{g}`") for g in groups] + txt = ", ".join(sorted(out)) + self._print(f" {txt}\n") + out = [ + ("%s (base class); used in %s" % (g, self._listing_category)) + for g in groups + ] + txt = ", ".join(out) + self._print(f".. index:: {txt}\n") + + # print full tree + self._print("**Structure**:\n") + for subnode in root.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, "file", subnode, optional, self._INDENTATION_UNIT, parent_path + ) # FIXME: +"/"+name ) + self._print_full_tree( + ns, root, nxclass_name, self._INDENTATION_UNIT, parent_path + ) + + self._print_anchor_list() + + # print NXDL source location + self._print("") + self._print("**NXDL Source**:") + self._print( + f" {REPO_URL}/{self._CATEGORY_TO_DIRNAME[category]}/{nxclass_name}.nxdl.xml" + ) + + return self._rst_lines + + def _print_anchor_list(self): + """Print the list of hypertext anchors.""" + if not self._anchor_registry: + return + anchors = self._anchor_registry.flush_anchor_buffer() + if not anchors: + return + + self._print("") + self._print("Hypertext Anchors") + self._print("-----------------\n") + self._print( + "List of hypertext anchors for all groups, fields,\n" + "attributes, and links defined in this class.\n\n" + ) + + def sorter(key): + return key.lower() + + rst = [f"* :ref:`{ref} <{ref}>`" for ref in sorted(anchors, key=sorter)] + + self._print("\n".join(rst)) + + @staticmethod + def _format_type(node): + typ = node.get("type", ":ref:`NX_CHAR `") # per default + if typ.startswith("NX_"): + typ = f":ref:`{typ} <{typ}>`" + return typ + + @staticmethod + def _format_units(node): + units = node.get("units", "") + if not units: + return "" + if units.startswith("NX_"): + units = rf"\ :ref:`{units} <{units}>`" + return f" {{units={units}}}" + + @staticmethod + def _get_doc_blocks(ns, node): + docnodes = node.xpath("nx:doc", namespaces=ns) + if docnodes is None or len(docnodes) == 0: + return "" + if len(docnodes) > 1: + raise Exception( + f"Too many doc elements: line {node.sourceline}, {Path(node.base).name}" + ) + docnode = docnodes[0] + + # be sure to grab _all_ content in the documentation + # it might look like XML + s = lxml.etree.tostring( + docnode, pretty_print=True, method="c14n", with_comments=False + ).decode("utf-8") + m = re.search(r"^]*>\n?(.*)\n?$", s, re.DOTALL) + if not m: + raise Exception(f"unexpected docstring [{s}] ") + text = m.group(1) + + # substitute HTML entities in markup: "<" for "<" + # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string + htmlparser = HTMLParser.HTMLParser() + try: # see #661 + import html + + text = html.unescape(text) + except (ImportError, AttributeError): + text = htmlparser.unescape(text) + + # Blocks are separated by whitelines + blocks = re.split("\n\\s*\n", text) + if len(blocks) == 1 and len(blocks[0].splitlines()) == 1: + return [blocks[0].rstrip().lstrip()] + + # Indentation must be given by first line + m = re.match(r"(\s*)(\S+)", blocks[0]) + if not m: + return [""] + indent = m.group(1) + + # Remove common indentation as determined from first line + if indent == "": + raise Exception( + "Missing initial indentation in of %s [%s]" + % (node.get("name"), blocks[0]) + ) + + out_blocks = [] + for block in blocks: + lines = block.rstrip().splitlines() + out_lines = [] + for line in lines: + if line[: len(indent)] != indent: + raise Exception( + 'Bad indentation in of %s [%s]: expected "%s" found "%s".' + % ( + node.get("name"), + block, + re.sub(r"\t", "\\\\t", indent), + re.sub(r"\t", "\\\\t", line), + ) + ) + out_lines.append(line[len(indent) :]) + out_blocks.append("\n".join(out_lines)) + return out_blocks + + def _get_doc_line(self, ns, node): + blocks = self._get_doc_blocks(ns, node) + if len(blocks) == 0: + return "" + if len(blocks) > 1: + raise Exception(f"Unexpected multi-paragraph doc [{'|'.join(blocks)}]") + return re.sub(r"\n", " ", blocks[0]) + + def _get_minOccurs(self, node): + """ + get the value for the ``minOccurs`` attribute + + :param obj node: instance of lxml.etree._Element + :returns str: value of the attribute (or its default) + """ + # TODO: can we improve on the default by examining nxdl.xsd? + minOccurs_default = str(int(self._use_application_defaults)) + minOccurs = node.get("minOccurs", minOccurs_default) + return minOccurs + + def _get_required_or_optional_text(self, node): + """ + make clear if a reported item is required or optional + + :param obj node: instance of lxml.etree._Element + :returns: formatted text + """ + tag = node.tag.split("}")[-1] + if tag in ("field", "group"): + optional_default = not self._use_application_defaults + optional = node.get("optional", optional_default) in (True, "true", "1", 1) + recommended = node.get("recommended", None) in (True, "true", "1", 1) + minOccurs = self._get_minOccurs(node) + if recommended: + optional_text = "(recommended) " + elif minOccurs in ("0", 0) or optional: + optional_text = "(optional) " + elif minOccurs in ("1", 1): + optional_text = "(required) " + else: + # this is unexpected and remarkable + # TODO: add a remark to the log + optional_text = f"(``minOccurs={str(minOccurs)}``) " + elif tag in ("attribute",): + optional_default = not self._use_application_defaults + optional = node.get("optional", optional_default) in (True, "true", "1", 1) + recommended = node.get("recommended", None) in (True, "true", "1", 1) + optional_text = {True: "(optional) ", False: "(required) "}[optional] + if recommended: + optional_text = "(recommended) " + else: + optional_text = "(unknown tag: " + str(tag) + ") " + return optional_text + + def _analyze_dimensions(self, ns, parent) -> str: + """These are the different dimensions that can occur: + + 1. Fixed rank + + + + + + + + 2. Variable rank because of optional dimensions + + + + + + + + + 3. Variable rank because no dimensions specified + + + + + The legacy way of doing this (still supported) + + + + + + 4. Rank and dimensions equal to that of another field called `field_name` + + + + + """ + node_list = parent.xpath("nx:dimensions", namespaces=ns) + if len(node_list) != 1: + return "" + node = node_list[0] + node_list = node.xpath("nx:dim", namespaces=ns) + + dims = [] + optional = False + for subnode in node_list: + # Dimension index (starts from index 1) + index = subnode.get("index", "") + if not index.isdigit(): + raise RuntimeError("A dimension must have an index") + index = int(index) + if index == 0: + # No longer needed: legacy way to specify that the + # rank is variable + continue + + # Expand dimensions when needed + index -= 1 + nadd = max(index - len(dims) + 1, 0) + if nadd: + dims += ["."] * nadd + + # Dimension symbol + dim = subnode.get("value") # integer or symbol from the table + if not dim: + ref = subnode.get("ref") + if ref: + return ( + f" (Rank: same as field {ref}, Dimensions: same as field {ref})" + ) + dim = "." # dimension has no symbol + + # Dimension might be optional + if subnode.get("required", "true").lower() == "false": + optional = True + elif optional: + raise RuntimeError( + "A required dimension cannot come after an optional dimension" + ) + if optional: + dim = f"[{dim}]" + + dims[index] = dim + + # When the rank is missing, set to the number of dimensions when + # there are dimensions specified and none of them are optional. + ndims = len(dims) + rank = node.get("rank", None) + if rank is None and not optional and ndims: + rank = str(ndims) + + # Validate rank and dimensions + rank_is_fixed = rank and rank.isdigit() + if optional and rank_is_fixed: + raise RuntimeError("A fixed rank cannot have optional dimensions") + if rank_is_fixed and ndims and int(rank) != ndims: + raise RuntimeError( + "The rank and the number of dimensions do not correspond" + ) + + # Omit rank and/or dimensions when not specified + if rank and dims: + dims = ", ".join(dims) + return f" (Rank: {rank}, Dimensions: [{dims}])" + elif rank: + return f" (Rank: {rank})" + elif dims: + dims = ", ".join(dims) + return f" (Dimensions: [{dims}])" + return "" + + def _hyperlink_target(self, parent_path, name, nxtype): + """Return internal hyperlink target for HTML anchor.""" + if nxtype == "attribute": + sep = "@" + else: + sep = "/" + target = f"{parent_path}{sep}{name}-{nxtype}" + if self._anchor_registry: + self._anchor_registry.add(target) + return f".. _{target}:\n" + + def _print_enumeration(self, indent, ns, parent): + node_list = parent.xpath("nx:item", namespaces=ns) + if len(node_list) == 0: + return "" + + if len(node_list) == 1: + self._print(f"{indent}Obligatory value:", end="") + else: + self._print(f"{indent}Any of these values:", end="") + + docs = OrderedDict() + for item in node_list: + name = item.get("value") + docs[name] = self._get_doc_line(ns, item) + + ENUMERATION_INLINE_LENGTH = 60 + + def show_as_typed_text(msg): + return f"``{msg}``" + + oneliner = " | ".join(map(show_as_typed_text, docs.keys())) + if ( + any(doc for doc in docs.values()) + or len(oneliner) > ENUMERATION_INLINE_LENGTH + ): + # print one item per line + self._print("\n") + for name, doc in docs.items(): + self._print(f"{indent} * {show_as_typed_text(name)}", end="") + if doc: + self._print(f": {doc}", end="") + self._print("\n") + else: + # print all items in one line + self._print(f" {oneliner}") + self._print("") + + def _print_doc(self, indent, ns, node, required=False): + blocks = self._get_doc_blocks(ns, node) + if len(blocks) == 0: + if required: + raise Exception("No documentation for: " + node.get("name")) + self._print("") + else: + for block in blocks: + for line in block.splitlines(): + self._print(f"{indent}{line}") + self._print() + + def _print_attribute(self, ns, kind, node, optional, indent, parent_path): + name = node.get("name") + index_name = name + self._print( + f"{indent}" f"{self._hyperlink_target(parent_path, name, 'attribute')}" + ) + self._print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") + self._print( + f"{indent}**@{name}**: {optional}{self._format_type(node)}{self._format_units(node)}\n" + ) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + node_list = node.xpath("nx:enumeration", namespaces=ns) + if len(node_list) == 1: + self._print_enumeration(indent + self._INDENTATION_UNIT, ns, node_list[0]) + + def _print_if_deprecated(self, ns, node, indent): + deprecated = node.get("deprecated", None) + if deprecated is not None: + self._print(f"\n{indent}.. index:: deprecated\n") + self._print(f"\n{indent}**DEPRECATED**: {deprecated}\n") + + def _print_full_tree(self, ns, parent, name, indent, parent_path): + """ + recursively print the full tree structure + + :param dict ns: dictionary of namespaces for use in XPath expressions + :param lxml_element_node parent: parent node to be documented + :param str name: name of elements, such as NXentry/NXuser + :param indent: to keep track of indentation level + :param parent_path: NX class path of parent nodes + """ + + self._use_application_defaults = self._listing_category in ( + "application definition", + "contributed definition", + ) + + for node in parent.xpath("nx:field", namespaces=ns): + name = node.get("name") + index_name = name + dims = self._analyze_dimensions(ns, node) + + optional_text = self._get_required_or_optional_text(node) + self._print(f"{indent}{self._hyperlink_target(parent_path, name, 'field')}") + self._print(f"{indent}.. index:: {index_name} (field)\n") + self._print( + f"{indent}**{name}**: " + f"{optional_text}" + f"{self._format_type(node)}" + f"{dims}" + f"{self._format_units(node)}" + "\n" + ) + + self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + node_list = node.xpath("nx:enumeration", namespaces=ns) + if len(node_list) == 1: + self._print_enumeration( + indent + self._INDENTATION_UNIT, ns, node_list[0] + ) + + for subnode in node.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, + "field", + subnode, + optional, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + for node in parent.xpath("nx:group", namespaces=ns): + name = node.get("name", "") + typ = node.get("type", "untyped (this is an error; please report)") + + optional_text = self._get_required_or_optional_text(node) + if typ.startswith("NX"): + if name == "": + name = typ.lstrip("NX").upper() + typ = f":ref:`{typ}`" + hTarget = self._hyperlink_target(parent_path, name, "group") + # target = hTarget.replace(".. _", "").replace(":\n", "") + # TODO: https://github.com/nexusformat/definitions/issues/1057 + self._print(f"{indent}{hTarget}") + self._print(f"{indent}**{name}**: {optional_text}{typ}\n") + + self._print_if_deprecated(ns, node, indent + self._INDENTATION_UNIT) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + for subnode in node.xpath("nx:attribute", namespaces=ns): + optional = self._get_required_or_optional_text(subnode) + self._print_attribute( + ns, + "group", + subnode, + optional, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + nodename = "%s/%s" % (name, node.get("type")) + self._print_full_tree( + ns, + node, + nodename, + indent + self._INDENTATION_UNIT, + parent_path + "/" + name, + ) + + for node in parent.xpath("nx:link", namespaces=ns): + name = node.get("name") + self._print(f"{indent}{self._hyperlink_target(parent_path, name, 'link')}") + self._print( + f"{indent}**{name}**: " + ":ref:`link` " + f"(suggested target: ``{node.get('target')}``" + "\n" + ) + self._print_doc(indent + self._INDENTATION_UNIT, ns, node) + + def _print(self, *args, end="\n"): + # TODO: change instances of \t to proper indentation + self._rst_lines.append(" ".join(args) + end) diff --git a/dev_tools/docs/nxdl_index.py b/dev_tools/docs/nxdl_index.py new file mode 100644 index 0000000000..98f90b95bc --- /dev/null +++ b/dev_tools/docs/nxdl_index.py @@ -0,0 +1,145 @@ +import os +from pathlib import Path +from typing import Dict + +import lxml.etree + +from ..globals import directories +from ..globals.nxdl import NXDL_NAMESPACE +from ..nxdl import iter_definitions + + +def nxdl_indices() -> Dict[str, dict]: + """For each directory under the NXDL root, create the content of an + index file which link all NeXus class definition doc files. + """ + indentation = " " * 4 + namespaces = {"nx": NXDL_NAMESPACE} + sections = dict() + root = directories.get_nxdl_root() + + for nxdl_file in iter_definitions(): + source = os.path.relpath(nxdl_file.parent, root) + section = sections.get(source) + if section is None: + preample = _DIRNAME_TO_PREAMBLE.get(source) + if preample is None: + continue + classes = [] + rst_lines = [] + index_file = directories.nxclass_build_root(nxdl_file) / "index.rst" + sections[source] = { + "index_file": index_file, + "rst_lines": rst_lines, + "classes": classes, + } + rst_lines.append(".. do NOT edit this file\n") + rst_lines.append(f" automatically generated by {__name__}\n") + rst_lines.append("\n") + rst_lines.append(preample) + rst_lines.append("\n") + else: + classes = sections[source]["classes"] + rst_lines = sections[source]["rst_lines"] + + nxclass_name = nxdl_file.with_suffix("").stem + classes.append(nxclass_name) + summary = get_nxclass_description(nxdl_file, namespaces) + rst_lines.append("\n") + rst_lines.append(f":ref:`{nxclass_name}`\n") + rst_lines.append(f"{indentation}{summary}\n") + + # Create a table of content for each index file + for section in sections.values(): + classes = section.pop("classes") + rst_lines = section["rst_lines"] + rst_lines.append("\n") + rst_lines.append(".. toctree::\n") + rst_lines.append(f"{indentation}:hidden:\n") + rst_lines.append("\n") + for cname in sorted(classes): + rst_lines.append(f"{indentation}{cname}\n") + + return sections + + +def get_nxclass_description(nxdl_file: Path, namespaces) -> str: + """ + get the summary line from each NXDL definition doc + + That's the first physical line of the doc string. + """ + tree = lxml.etree.parse(nxdl_file) + root = tree.getroot() + nodes = root.xpath("nx:doc", namespaces=namespaces) + if len(nodes) != 1: + raise RuntimeError(f"wrong number of nodes in NXDL: {nxdl_file}") + return nodes[0].text.strip().splitlines()[0] + + +_DIRNAME_TO_PREAMBLE = { + "base_classes": """ +.. index:: + ! see: class definitions; base class + ! base class + +.. _base.class.definitions: + +Base Class Definitions +###################### + +A description of each NeXus base class definition is given. +NeXus base class definitions define the set of terms that +*might* be used in an instance of that class. +Consider the base classes as a set of *components* +that are used to construct a data file. + """, + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - + "applications": """ +.. index:: + ! see: class definitions; application definition + ! application definition + +.. _application.definitions: + +Application Definitions +######################### + +A description of each NeXus application definition is given. +NeXus application definitions define the *minimum* +set of terms that +*must* be used in an instance of that class. +Application definitions also may define terms that +are optional in the NeXus data file. The definition, in this case, +reserves the exact term by declaring its spelling and description. +Consider an application definition as a *contract* +between a data provider (such as the beam line control system) and a +data consumer (such as a data analysis program for a scientific technique) +that describes the information is certain to be available in a data file. + +Use NeXus links liberally in data files to reduce duplication of data. +In application definitions involving raw data, +write the raw data in the :ref:`NXinstrument` tree and then link to it +from the location(s) defined in the relevant application definition. + """, + # - - - - - - - - - - - - - - - - - - - - - - - - - - - - + "contributed_definitions": """ +.. index:: + ! see: class definitions; contributed definition + ! contributed definition + +.. _contributed.definitions: + +Contributed Definitions +######################### + +A description of each NeXus contributed definition is given. +NXDL files in the NeXus contributed definitions include propositions from +the community for NeXus base classes or application definitions, as well +as other NXDL files for long-term archival by NeXus. Consider the contributed +definitions as either in *incubation* or a special +case not for general use. The :ref:`NIAC` is charged to review any new contributed +definitions and provide feedback to the authors before ratification +and acceptance as either a base class or application definition. + """, +} diff --git a/dev_tools/docs/xsd.py b/dev_tools/docs/xsd.py new file mode 100644 index 0000000000..c0936416d1 --- /dev/null +++ b/dev_tools/docs/xsd.py @@ -0,0 +1,503 @@ +import textwrap +from pathlib import Path +from typing import List + +import lxml.etree + +from ..globals import directories +from ..globals.errors import NXDLParseError +from ..globals.nxdl import XSD_NAMESPACE +from ..utils.types import PathLike + + +class XSDDocGenerator: + """ + Read the NXDL field types specification and find + all the valid data types. Write a restructured + text (.rst) document for use in the NeXus manual in + the NXDL chapter. + """ + + TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles + INDENTATION = " " * 4 + DATATYPE_DICT = { + "basicComponent": """/xs:schema//xs:complexType[@name='basicComponent']""", + "validItemName": """/xs:schema//xs:simpleType[@name='validItemName']""", + "validNXClassName": """/xs:schema//xs:simpleType[@name='validNXClassName']""", + "validTargetName": """/xs:schema//xs:simpleType[@name='validTargetName']""", + "nonNegativeUnbounded": """/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']""", + } + + def __init__(self) -> None: + self.ns = {"xs": XSD_NAMESPACE} + self._rst_lines = None + + def __call__(self, xsd_file: PathLike) -> List[str]: + self._rst_lines = list() + xsd_file = Path(xsd_file) + try: + self._parse_xsd_file(xsd_file) + except Exception: + raise NXDLParseError(xsd_file) + return self._rst_lines + + def _parse_xsd_file(self, xsd_file: Path): + tree = lxml.etree.parse(str(xsd_file)) + + self._print(f".. auto-generated by {__name__} -- DO NOT EDIT") + self._print(ELEMENT_PREAMBLE) + + for name in sorted(ELEMENT_DICT): + self._print("") + self._print(".. index:: ! %s (NXDL element)\n" % name) + self._print(".. _%s:\n" % name) + self.print_title(name, indentLevel=0) + self._print("\n") + self._print(ELEMENT_DICT[name]) + self._print("\n") + self.add_figure(name, indentLevel=0) + + self._print(DATATYPE_PREAMBLE) + + path_list = ( + "/xs:schema/xs:complexType[@name='attributeType']", + "/xs:schema/xs:element[@name='definition']", + "/xs:schema/xs:complexType[@name='definitionType']", + "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", + "/xs:schema/xs:complexType[@name='dimensionsType']", + "/xs:schema/xs:complexType[@name='docType']", + "/xs:schema/xs:complexType[@name='enumerationType']", + "/xs:schema/xs:complexType[@name='fieldType']", + "/xs:schema/xs:complexType[@name='choiceType']", + "/xs:schema/xs:complexType[@name='groupType']", + "/xs:schema/xs:complexType[@name='linkType']", + "/xs:schema/xs:complexType[@name='symbolsType']", + "/xs:schema/xs:complexType[@name='basicComponent']", + "/xs:schema/xs:simpleType[@name='validItemName']", + "/xs:schema/xs:simpleType[@name='validNXClassName']", + "/xs:schema/xs:simpleType[@name='validTargetName']", + "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", + ) + for path in path_list: + nodes = self.pick_nodes_from_xpath(tree, path) + self._print("\n.. Xpath = %s\n" % path) + self.general_handler(parent=nodes[0]) + + self._print(DATATYPE_POSTAMBLE) + + def _tag_match(self, parent, match_list): + """match this tag to a list""" + if parent is None: + raise ValueError("Must supply a valid parent node") + parent_tag = parent.tag + tag_found = False + for item in match_list: + # this routine only handles certain XML Schema components + tag_found = parent_tag == "{%s}%s" % (self.ns["xs"], item) + if tag_found: + break + return tag_found + + def _indent(self, indentLevel): + return self.INDENTATION * indentLevel + + def print_title(self, title, indentLevel): + self._print(title) + self._print(self.TITLE_MARKERS[indentLevel] * len(title) + "\n") + + def general_handler(self, parent=None, indentLevel=0): + """Handle XML nodes like the former XSLT template""" + # ignore things we don't know how to handle + known_tags = ("complexType", "simpleType", "group", "element", "attribute") + if not self._tag_match(parent, known_tags): + return + + parent_name = parent.get("name") + if parent_name is None: + return + + simple_tag = parent.tag[ + parent.tag.find("}") + 1 : + ] # cut off the namespace identifier + + # ... + name = parent_name # + ' data type' + if simple_tag == "attribute": + name = "@" + name + + if indentLevel == 0 and simple_tag not in ("attribute"): + self._print(f".. index:: ! {name} (NXDL data type)\n") + self._print(f"\n.. _NXDL.data.type.{name}:\n") + + self.print_title(name, indentLevel) + + self.print_docs(parent, indentLevel) + + if len(parent.xpath("xs:attribute", namespaces=self.ns)) > 0: + self.print_title("Attributes of " + name, indentLevel + 1) + self.apply_templates(parent, "xs:attribute", indentLevel + 1) + + node_list = parent.xpath("xs:restriction", namespaces=self.ns) + if len(node_list) > 0: + # print_title("Restrictions of "+name, indentLevel+1) + self.restriction_handler(node_list[0], indentLevel + 1) + node_list = parent.xpath( + "xs:simpleType/xs:restriction/xs:enumeration", namespaces=self.ns + ) + if len(node_list) > 0: + # print_title("Enumerations of "+name, indentLevel+1) + self.apply_templates( + parent, + "xs:simpleType/xs:restriction", + indentLevel + 1, + handler=self.restriction_handler, + ) + + if len(parent.xpath("xs:sequence/xs:element", namespaces=self.ns)) > 0: + self.print_title("Elements of " + name, indentLevel + 1) + self.apply_templates(parent, "xs:sequence/xs:element", indentLevel + 1) + + node_list = parent.xpath("xs:sequence/xs:group", namespaces=self.ns) + if len(node_list) > 0: + self.print_title("Groups under " + name, indentLevel + 1) + self.print_docs(node_list[0], indentLevel + 1) + + self.apply_templates(parent, "xs:simpleType", indentLevel + 1) + self.apply_templates(parent, "xs:complexType", indentLevel + 1) + self.apply_templates(parent, "xs:complexType/xs:attribute", indentLevel + 1) + self.apply_templates( + parent, "xs:complexContent/xs:extension/xs:attribute", indentLevel + 1 + ) + self.apply_templates( + parent, "xs:complexType/xs:sequence/xs:attribute", indentLevel + 1 + ) + self.apply_templates( + parent, "xs:complexType/xs:sequence/xs:element", indentLevel + 1 + ) + self.apply_templates( + parent, + "xs:complexContent/xs:extension/xs:sequence/xs:element", + indentLevel + 1, + ) + + def restriction_handler(self, parent=None, indentLevel=0): + """Handle XSD restriction nodes like the former XSLT template""" + if not self._tag_match(parent, ("restriction",)): + return + self.print_docs(parent, indentLevel) + self._print("\n") + self._print(self._indent(indentLevel) + "The value may be any") + base = parent.get("base") + pattern_nodes = parent.xpath("xs:pattern", namespaces=self.ns) + enumeration_nodes = parent.xpath("xs:enumeration", namespaces=self.ns) + if len(pattern_nodes) > 0: + self._print( + self._indent(indentLevel) + + "``%s``" % base + + " that *also* matches the regular expression::\n" + ) + self._print( + self._indent(indentLevel) + " " * 4 + pattern_nodes[0].get("value") + ) + elif len(pattern_nodes) > 0: + # how will this be reached? Perhaps a deprecated procedure + self._print( + self._indent(indentLevel) + "``%s``" % base + " from this list:" + ) + for node in enumeration_nodes: + self.enumeration_handler(node, indentLevel) + self.print_docs(node, indentLevel) + self._print(self._indent(indentLevel)) + elif len(enumeration_nodes) > 0: + self._print(self._indent(indentLevel) + "one from this list only:\n") + for node in enumeration_nodes: + self.enumeration_handler(node, indentLevel) + self.print_docs(parent, indentLevel) + self._print(self._indent(indentLevel)) + else: + self._print("@" + base) + self._print("\n") + + def enumeration_handler(self, parent=None, indentLevel=0): + """Handle XSD enumeration nodes like the former XSLT template""" + if not self._tag_match(parent, ["enumeration"]): + return + self._print(self._indent(indentLevel) + "* ``%s``" % parent.get("value")) + self.print_docs(parent, indentLevel) + + def apply_templates(self, parent, path, indentLevel, handler=None): + """iterate the nodes found on the supplied XPath expression""" + if handler is None: + handler = self.general_handler + db = {} + for node in parent.xpath(path, namespaces=self.ns): + name = node.get("name") or node.get("ref") or node.get("value") + if name is not None: + if name in ("nx:groupGroup",): + self._print(">" * 45, name) + if name in db: + raise KeyError("Duplicate name found: " + name) + db[name] = node + for name in sorted(db): + node = db[name] + handler(node, indentLevel) + # self.print_docs(node, indentLevel) + + def print_docs(self, parent, indentLevel=0): + docs = self.get_doc_from_node(parent) + if docs is not None: + self._print(self._indent(indentLevel) + "\n") + for line in docs.splitlines(): + self._print(self._indent(indentLevel) + line) + self._print(self._indent(indentLevel) + "\n") + + def get_doc_from_node(self, node, retval=None): + annotation_node = node.find("xs:annotation", self.ns) + if annotation_node is None: + return retval + documentation_node = annotation_node.find("xs:documentation", self.ns) + if documentation_node is None: + return retval + + # Be sure to grab _all_ content in the node. + # In the documentation nodes, use XML entities ("<"" instead of "<") + # for documentation characters that would otherwise be considered as XML. + s = lxml.etree.tostring(documentation_node, method="text", pretty_print=True) + rst = s.decode().lstrip("\n") # remove any leading blank lines + rst = rst.rstrip() # remove any trailing white space + text = textwrap.dedent(rst) # remove common leading space + + # substitute HTML entities in markup: "<" for "<" + # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string + try: # see #661 + import html + + text = html.unescape(text) + except (ImportError, AttributeError): + from html import parser as HTMLParser + + htmlparser = HTMLParser.HTMLParser() + text = htmlparser.unescape(text) + + return text.lstrip() + + def add_figure(self, name, indentLevel=0): + imageFile = f"img/nxdl/nxdl_{name}.png" + figure_id = f"fig.nxdl_{name}" + file_name = directories.manual_source_sphinxsroot() / imageFile + if not file_name.exists(): + return + text = FIGURE_FMT % ( + figure_id, + imageFile, + name, + "80%", + name, + name, + ) + indent = self._indent(indentLevel) + for line in text.splitlines(): + self._print(indent + line) + self._print("\n") + + def pick_nodes_from_xpath(self, parent, path): + return parent.xpath(path, namespaces=self.ns) + + def _print(self, *args, end="\n"): + # TODO: change instances of \t to proper indentation + self._rst_lines.append(" ".join(args) + end) + + +ELEMENT_DICT = { + "attribute": """ +An ``attribute`` element can *only* be a child of a +``field`` or ``group`` element. +It is used to define *attribute* elements to be used and their data types +and possibly an enumeration of allowed values. + +For more details, see: +:ref:`NXDL.data.type.attributeType` + """, + "definition": """ +A ``definition`` element can *only* be used +at the root level of an NXDL specification. +Note: Due to the large number of attributes of the ``definition`` element, +they have been omitted from the figure below. + +For more details, see: +:ref:`NXDL.data.type.definition`, +:ref:`NXDL.data.type.definitionType`, and +:ref:`NXDL.data.type.definitionTypeAttr` + """, + "dimensions": """ +The ``dimensions`` element describes the *shape* of an array. +It is used *only* as a child of a ``field`` element. + +For more details, see: +:ref:`NXDL.data.type.dimensionsType` + """, + "doc": """ +A ``doc`` element can be a child of most NXDL elements. In most cases, the +content of the ``doc`` element will also become part of the NeXus manual. + +:element: {any}: + +In documentation, it may be useful to +use an element that is not directly specified by the NXDL language. +The *any* element here says that one can use any element +at all in a ``doc`` element and NXDL will not process it but pass it through. + +For more details, see: +:ref:`NXDL.data.type.docType` + """, + "enumeration": """ +An ``enumeration`` element can *only* be a child of a +``field`` or ``attribute`` element. +It is used to restrict the available choices to a predefined list, +such as to control varieties in spelling of a controversial word (such as +*metre* vs. *meter*). + +For more details, see: +:ref:`NXDL.data.type.enumerationType` + """, + "field": """ +The ``field`` element provides the value of a named item. Many different attributes +are available to further define the ``field``. Some of the attributes are not +allowed to be used together (such as ``axes`` and ``axis``); see the documentation +of each for details. +It is used *only* as a child of a ``group`` element. + +For more details, see: +:ref:`NXDL.data.type.fieldType` + """, + "choice": """ +A ``choice`` element is used when a named group might take one +of several possible NeXus base classes. Logically, it must +have at least two group children. + +For more details, see: +:ref:`NXDL.data.type.choiceType` + """, + "group": """ +A ``group`` element can *only* be a child of a +``definition`` or ``group`` element. +It describes a common level of organization in a NeXus data file, similar +to a subdirectory in a file directory tree. + +For more details, see: +:ref:`NXDL.data.type.groupType` + """, + "link": """ +.. index:: + single: link target + +A ``link`` element can *only* be a child of a +``definition``, +``field``, or ``group`` element. +It describes the path to the original source of the parent +``definition``, +``field``, or ``group``. + +For more details, see: +:ref:`NXDL.data.type.linkType` + """, + "symbols": """ +A ``symbols`` element can *only* be a child of a ``definition`` element. +It defines the array index symbols to be used when defining arrays as +``field`` elements with common dimensions and lengths. + +For more details, see: +:ref:`NXDL.data.type.symbolsType` + """, +} + + +ELEMENT_PREAMBLE = """ +============================= +NXDL Elements and Field Types +============================= + +The documentation in this section has been obtained directly +from the NXDL Schema file: *nxdl.xsd*. +First, the basic elements are defined in alphabetical order. +Attributes to an element are indicated immediately following the element +and are preceded with an "@" symbol, such as +**@attribute**. +Then, the common data types used within the NXDL specification are defined. +Pay particular attention to the rules for *validItemName* +and *validNXClassName*. + +.. + 2010-11-29,PRJ: + This contains a lot of special case code to lay out the NXDL chapter. + It could be cleaner but that would also involve some cooperation on + anyone who edits nxdl.xsd which is sure to break. The special case ensures + the parts come out in the chosen order. BUT, it is possible that new + items in nxdl.xsd will not automatically go in the manual. + Can this be streamlined with some common methods? + Also, there is probably too much documentation in nxdl.xsd. Obscures the function. + +.. index:: + see:attribute; NXDL attribute + ! single: NXDL elements + +.. _NXDL.elements: + +NXDL Elements +============= + + """ + +DATATYPE_PREAMBLE = """ + +.. _NXDL.data.types.internal: + +NXDL Field Types (internal) +=========================== + +Field types that define the NXDL language are described here. +These data types are defined in the XSD Schema (``nxdl.xsd``) +and are used in various parts of the Schema to define common structures +or to simplify a complicated entry. While the data types are not intended for +use in NXDL specifications, they define structures that may be used in NXDL specifications. + +""" + +DATATYPE_POSTAMBLE = """ +**The** ``xs:string`` **data type** + The ``xs:string`` data type can contain characters, + line feeds, carriage returns, and tab characters. + See https://www.w3schools.com/xml/schema_dtypes_string.asp + for more details. + +**The** ``xs:token`` **data type** + The ``xs:string`` data type is derived from the + ``xs:string`` data type. + + The ``xs:token`` data type also contains characters, + but the XML processor will remove line feeds, carriage returns, tabs, + leading and trailing spaces, and multiple spaces. + See https://www.w3schools.com/xml/schema_dtypes_string.asp + for more details. +""" + + +FIGURE_FMT = """ +.. compound:: + + .. _%s: + + .. figure:: %s + :alt: fig.nxdl/nxdl_%s + :width: %s + + Graphical representation of the NXDL ``%s`` element + + .. Images of NXDL structure are generated from nxdl.xsd source + using the Eclipse XML Schema Editor (Web Tools Platform). Open the nxdl.xsd file and choose the + "Design" tab. Identify the structure to be documented and double-click to expand + as needed to show the detail. Use the XSD > "Export Diagram as Image ..." menu item (also available + as button in top toolbar). + Set the name: "nxdl_%s.png" and move the file into the correct location using + your operating system's commands. Commit the revision to version control. +""" diff --git a/dev_tools/docs/xsd_units.py b/dev_tools/docs/xsd_units.py new file mode 100644 index 0000000000..63d7714289 --- /dev/null +++ b/dev_tools/docs/xsd_units.py @@ -0,0 +1,72 @@ +""" +Read the the NeXus NXDL types specification and find +all the valid types of units. Write a restructured +text (.rst) document for use in the NeXus manual in +the NXDL chapter. +""" + + +from pathlib import Path +from typing import List + +import lxml.etree + +from ..globals.nxdl import XSD_NAMESPACE +from ..utils.types import PathLike + + +def generate_xsd_units_doc( + xsd_file: PathLike, nodeMatchString: str, section: str +) -> List[str]: + xsd_file = str(Path(xsd_file)) + tree = lxml.etree.parse(xsd_file) + + rst_lines = [f".. auto-generated by {__name__} -- DO NOT EDIT\n"] + rst_lines.append("\n") + + rst_lines.append(f".. nodeMatchString : {nodeMatchString}\n") + rst_lines.append("\n") + + db = {} + ns = {"xs": XSD_NAMESPACE} + node_list = tree.xpath("//xs:simpleType", namespaces=ns) + + # get the names of all the types of units + members = [] + for node in node_list: + if node.get("name") == nodeMatchString: + union = node.xpath("xs:union", namespaces=ns) + members = union[0].get("memberTypes", "").split() + + # get the definition of each type of units + for node in node_list: + node_name = node.get("name") + if node_name is None: + continue + if "nxdl:" + node_name in members: + words = node.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] + examples = [] + for example in words.iterchildren(): + nm = example.attrib.get("name") + if nm is not None and nm == "example": + examples.append("``" + example.text + "``") + a = words.text + if len(examples) > 0: + a = " ".join(a.split()) + ",\n\texample(s): " + " | ".join(examples) + db[node_name] = a + + # for item in node.xpath("xs:restriction//xs:enumeration", namespaces=ns): + # key = "%s" % item.get("value") + # words = item.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] + # db[key] = words.text + + # this list is too long to make this a table in latex + # for two columns, a Sphinx fieldlist will do just as well + for key in sorted(db): + rst_lines.append(f".. index:: ! {key} ({section} type)\n\n") # index entry + rst_lines.append(f".. _{key}:\n\n") # cross-reference point + rst_lines.append(f":{key}:\n") + for line in db[key].splitlines(): + rst_lines.append(f" {line}\n") + rst_lines.append("\n") + return rst_lines diff --git a/dev_tools/globals/__init__.py b/dev_tools/globals/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/globals/directories.py b/dev_tools/globals/directories.py new file mode 100644 index 0000000000..a80f2f1ac5 --- /dev/null +++ b/dev_tools/globals/directories.py @@ -0,0 +1,115 @@ +import os +from pathlib import Path + +from ..utils.types import PathLike + + +def get_source_root() -> Path: + """Root directory of the source code for the documentation + (git repository root by default)""" + if _SOURCE_ROOT is None: + set_source_root(Path(__file__).absolute().parent.parent.parent) + return _SOURCE_ROOT + + +def set_source_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + global _SOURCE_ROOT + _SOURCE_ROOT = root.absolute() + + +def get_nxdl_root() -> Path: + """Root directory of the XSD and NXDL files + (same as the source root by default)""" + if _NXDL_ROOT is None: + set_nxdl_root(get_source_root()) + return _NXDL_ROOT + + +def set_nxdl_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + assert (root / _XSD_FILE_NAME).exists() + global _NXDL_ROOT + _NXDL_ROOT = root.absolute() + + +def get_xsd_file() -> Path: + """The XSD file that defines the NXDL""" + return get_nxdl_root() / _XSD_FILE_NAME + + +def get_xsd_units_file() -> Path: + """The XSD file that defines the units in the NXDL""" + return get_nxdl_root() / _XSD_UNITS_FILE_NAME + + +def get_nxdl_version_file() -> Path: + """The version of the NeXus standard and the NeXus Definition language""" + return get_source_root() / _VERSION_FILE_NAME + + +def get_build_root() -> Path: + """Root directory in which the sources for documentation + building are generated.""" + if _BUILD_ROOT is None: + set_build_root(get_source_root() / "build") + return _BUILD_ROOT + + +def set_build_root(root: PathLike) -> None: + if not isinstance(root, Path): + root = Path(root) + global _BUILD_ROOT + _BUILD_ROOT = root.absolute() + + +def manual_source_root() -> Path: + """Source directory of the NeXus User Manual""" + return get_source_root() / "manual" + + +def manual_build_root() -> Path: + """Build directory of the NeXus User Manual""" + return get_build_root() / "manual" + + +def impatient_source_root() -> Path: + """Source directory of the NeXus Impatient Guide""" + return get_source_root() / "impatient-guide" + + +def impatient_build_root() -> Path: + """Build directory of the NeXus Impatient Guide""" + return get_build_root() / "impatient-guide" + + +def manual_source_sphinxsroot() -> Path: + """Sphinx source directory of the NeXus User Manual""" + return manual_source_root() / "source" + + +def manual_build_sphinxsroot() -> Path: + """Sphinx source directory for building of the NeXus User Manual""" + return manual_build_root() / "source" + + +def manual_build_staticroot() -> Path: + """Static source directory for building of the NeXus User Manual""" + return manual_build_sphinxsroot() / "_static" + + +def nxclass_build_root(nxdl_file: Path) -> Path: + """NeXus class documentation directory for building of the NeXus User Manual""" + root = manual_build_sphinxsroot() / "classes" + root /= os.path.relpath(nxdl_file.parent, get_nxdl_root()) + return root + + +_XSD_FILE_NAME = "nxdl.xsd" +_XSD_UNITS_FILE_NAME = "nxdlTypes.xsd" +_VERSION_FILE_NAME = "NXDL_VERSION" +_NXDL_ROOT = None +_BUILD_ROOT = None +_SOURCE_ROOT = None diff --git a/dev_tools/globals/errors.py b/dev_tools/globals/errors.py new file mode 100644 index 0000000000..7b15e9fa60 --- /dev/null +++ b/dev_tools/globals/errors.py @@ -0,0 +1,14 @@ +class XMLSyntaxError(Exception): + """XML file has a syntax error""" + + pass + + +class NXDLSyntaxError(XMLSyntaxError): + """XML file written in NXDL has a syntax error""" + + pass + + +class NXDLParseError(Exception): + """XML file written in NXDL cannot be parsed""" diff --git a/dev_tools/globals/nxdl.py b/dev_tools/globals/nxdl.py new file mode 100644 index 0000000000..eb4013e2cf --- /dev/null +++ b/dev_tools/globals/nxdl.py @@ -0,0 +1,21 @@ +"""Namespace of the schema in which NXDL is defined +(the XSD namespace) and the namespace of the NeXus +class definitions (the NXDL namespace). + +Namespaces are URL's solely to be globally unique. +Do not use these URL's for validation. That's what +"xsi:schemaLocation" is for. Currently this is the +relative URI "../nxdl.xsd" which means validation +can only be done for NXDL files in subdirectories. +""" + +from .directories import get_nxdl_version_file + +XSD_NAMESPACE = "http://www.w3.org/2001/XMLSchema" +NXDL_NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" + + +def get_nxdl_version() -> str: + """The version of the NeXus standard and the NeXus Definition language""" + with open(get_nxdl_version_file(), "r") as fh: + return fh.read().strip() diff --git a/dev_tools/globals/urls.py b/dev_tools/globals/urls.py new file mode 100644 index 0000000000..3d70950556 --- /dev/null +++ b/dev_tools/globals/urls.py @@ -0,0 +1,2 @@ +REPO_URL = "https://github.com/nexusformat/definitions/blob/main" +MANUAL_URL = "https://manual.nexusformat.org/" diff --git a/dev_tools/nxdl/__init__.py b/dev_tools/nxdl/__init__.py new file mode 100644 index 0000000000..33b2684778 --- /dev/null +++ b/dev_tools/nxdl/__init__.py @@ -0,0 +1,10 @@ +"""NeXus classes are defined in the NeXus Definition Language (NXDL). + +The NeXus Definition Language is an XML Schema Definition (XSD). +Each NeXus class is defined in an XML file using the NXDL schema. +""" + +from .discover import find_definition # noqa F401 +from .discover import iter_definitions # noqa F401 +from .syntax import nxdl_schema # noqa F401 +from .syntax import validate_definition # noqa F401 diff --git a/dev_tools/nxdl/discover.py b/dev_tools/nxdl/discover.py new file mode 100644 index 0000000000..6f64425a73 --- /dev/null +++ b/dev_tools/nxdl/discover.py @@ -0,0 +1,30 @@ +from pathlib import Path +from typing import Iterator +from typing import Optional +from typing import Tuple + +from ..globals.directories import get_build_root +from ..globals.directories import get_nxdl_root + + +def iter_definitions(*subdirs: Tuple[str]) -> Iterator[Path]: + """Yield all NeXus class definitions""" + root = get_nxdl_root() + build_root = get_build_root() + for subdir in subdirs: + root /= subdir + for path in sorted(root.rglob("*.nxdl.xml")): + try: + path.relative_to(build_root) + # Skip file in the build root + continue + except ValueError: + yield path + + +def find_definition(nxclass: str, *subdirs: Tuple[str]) -> Optional[Path]: + """Find the definition of a NeXus class""" + s = nxclass.lower() + ".nxdl" + for path in iter_definitions(*subdirs): + if path.stem.lower() == s: + return path diff --git a/dev_tools/nxdl/syntax.py b/dev_tools/nxdl/syntax.py new file mode 100644 index 0000000000..e5b37b657e --- /dev/null +++ b/dev_tools/nxdl/syntax.py @@ -0,0 +1,28 @@ +from typing import Optional + +import lxml.etree + +from ..globals import errors +from ..globals.directories import get_xsd_file +from ..utils.types import PathLike + + +def nxdl_schema() -> lxml.etree.XMLSchema: + return lxml.etree.XMLSchema(lxml.etree.parse(get_xsd_file())) + + +def validate_definition( + xml_file_name: PathLike, + xml_schema: Optional[lxml.etree.XMLSchema] = None, +): + xml_file_name = str(xml_file_name) + try: + xml_tree = lxml.etree.parse(xml_file_name) + except lxml.etree.XMLSyntaxError: + raise errors.XMLSyntaxError(xml_file_name) + if xml_schema is None: + xml_schema = nxdl_schema() + try: + xml_schema.assertValid(xml_tree) + except lxml.etree.DocumentInvalid: + raise errors.NXDLSyntaxError(xml_file_name) diff --git a/dev_tools/requirements.txt b/dev_tools/requirements.txt new file mode 100644 index 0000000000..815406521c --- /dev/null +++ b/dev_tools/requirements.txt @@ -0,0 +1,14 @@ +# Prepare for Documentation +lxml +pyyaml + +# Documentation building +sphinx>=5 + +# Testing +pytest + +# Code style and auto-formatting +black>=22.3 +flake8>=4 +isort>=5.10 \ No newline at end of file diff --git a/dev_tools/setup.cfg b/dev_tools/setup.cfg new file mode 100644 index 0000000000..4582c6c60b --- /dev/null +++ b/dev_tools/setup.cfg @@ -0,0 +1,8 @@ +# E501 (line too long) ignored for now +# E203 and W503 incompatible with black formatting (https://black.readthedocs.io/en/stable/compatible_configs.html#flake8) +[flake8] +ignore = E501, E203, W503 +max-line-length = 88 + +[isort] +force_single_line = true \ No newline at end of file diff --git a/dev_tools/tests/__init__.py b/dev_tools/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/tests/test_docs.py b/dev_tools/tests/test_docs.py new file mode 100644 index 0000000000..f0a8bca984 --- /dev/null +++ b/dev_tools/tests/test_docs.py @@ -0,0 +1,52 @@ +import pytest + +from ..docs import AnchorRegistry +from ..docs import NXClassDocGenerator +from ..docs import XSDDocGenerator +from ..docs import nxdl_indices +from ..globals.directories import get_xsd_file +from ..nxdl import iter_definitions + + +@pytest.fixture(scope="module") +def doc_generator(): + return NXClassDocGenerator() + + +@pytest.fixture(scope="module") +def anchor_registry(): + return AnchorRegistry() + + +@pytest.fixture(scope="module") +def anchor_registry_write(tmpdir_factory): + tmpdir = tmpdir_factory.mktemp("registry") + reg = AnchorRegistry(output_path=tmpdir) + yield reg + reg.write() + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_generate_doc(nxdl_file, doc_generator): + assert doc_generator(nxdl_file) + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_anchor_list(nxdl_file, doc_generator, anchor_registry): + assert doc_generator(nxdl_file, anchor_registry=anchor_registry) + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_anchor_write_list(nxdl_file, doc_generator, anchor_registry_write): + assert doc_generator(nxdl_file, anchor_registry=anchor_registry_write) + + +def test_nxdl_indices(): + sections = nxdl_indices() + expected = {"base_classes", "applications", "contributed_definitions"} + assert set(sections) == expected + + +def test_xsd_generate_doc(): + generator = XSDDocGenerator() + assert generator(get_xsd_file()) diff --git a/dev_tools/tests/test_nxdl.py b/dev_tools/tests/test_nxdl.py new file mode 100644 index 0000000000..bdeafe124a --- /dev/null +++ b/dev_tools/tests/test_nxdl.py @@ -0,0 +1,30 @@ +import pytest + +from ..nxdl import find_definition +from ..nxdl import iter_definitions +from ..nxdl import nxdl_schema +from ..nxdl import validate_definition + + +def test_iter_definitions(): + all_files = set(iter_definitions()) + assert all_files + base_files = set(iter_definitions("base_classes")) + assert base_files + assert not (base_files - all_files) + + +def test_find_definition(): + assert find_definition("NXroot") + assert not find_definition("NXwrong") + assert not find_definition("NXroot", "applications") + + +@pytest.fixture(scope="module") +def xml_schema(): + return nxdl_schema() + + +@pytest.mark.parametrize("nxdl_file", list(iter_definitions())) +def test_nxdl_syntax(nxdl_file, xml_schema): + validate_definition(nxdl_file, xml_schema) diff --git a/dev_tools/utils/__init__.py b/dev_tools/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/dev_tools/utils/copy.py b/dev_tools/utils/copy.py new file mode 100644 index 0000000000..472c4b4ad0 --- /dev/null +++ b/dev_tools/utils/copy.py @@ -0,0 +1,50 @@ +import shutil +from pathlib import Path +from typing import List +from typing import Tuple + +import requests + +from ..globals import directories + + +def copyfile(from_path: Path, to_path: Path) -> None: + print("copy", from_path) + print(" ->", to_path) + shutil.copyfile(from_path, to_path) + + +def copydir(from_path: Path, to_path: Path) -> None: + print("copy", from_path) + print(" ->", to_path) + shutil.copytree(from_path, to_path, dirs_exist_ok=True) + + +def download(url: str, to_path: Path) -> None: + print("download", url) + print(" ->", to_path) + r = requests.get(url) + with open(to_path, "wb") as fh: + fh.write(r.content) + + +def copy_files(files: List[Tuple[str, str, bool]]) -> None: + source_root = directories.get_source_root() + build_root = directories.get_build_root() + for from_subname, to_subname, overwrite in files: + to_path = build_root / to_subname + if overwrite or not to_path.exists(): + from_path = source_root / from_subname + copyfile(from_path, to_path) + else: + print("already exists", to_path) + + +def download_files(urls: List[Tuple[str, str, bool]]) -> None: + build_root = directories.get_build_root() + for url, subname, overwrite in urls: + to_path = build_root / subname + if overwrite or not to_path.exists(): + download(url, to_path) + else: + print("already exists", to_path) diff --git a/dev_tools/utils/diff.py b/dev_tools/utils/diff.py new file mode 100644 index 0000000000..7269ab1a18 --- /dev/null +++ b/dev_tools/utils/diff.py @@ -0,0 +1,28 @@ +import difflib +import tempfile +from pathlib import Path +from typing import List + + +def diff_ascii(src_file: Path, new_content: List[str], dest_file: Path) -> None: + """`new_content` is the parsed content of `src_file` to be compared + with the content of `dest_file`. + """ + if dest_file.exists(): + with open(dest_file, "r") as fh: + original_content = list(fh) + else: + original_content = list() + + with tempfile.TemporaryFile("w+") as fh: + fh.writelines(new_content) + fh.seek(0) + new_content = list(fh) + + for line in difflib.unified_diff( + original_content, + new_content, + fromfile=str(dest_file), + tofile=str(src_file), + ): + print(line) diff --git a/dev_tools/utils/types.py b/dev_tools/utils/types.py new file mode 100644 index 0000000000..a2f5bbf6ff --- /dev/null +++ b/dev_tools/utils/types.py @@ -0,0 +1,4 @@ +import os +from typing import Union + +PathLike = Union[str, os.PathLike] From 46cc112c4f3ee5e68c115d5772e8b85e60e7af98 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Tue, 28 Jun 2022 10:23:28 +0200 Subject: [PATCH 2/6] use dev_tools and Makefile and remove obsolete files --- .github/workflows/ci.yaml | 29 +- .github/workflows/publish-sphinx.yml | 91 -- .github/workflows/syntax-checks.yml | 47 - .gitignore | 10 - BUILDING.rst | 82 +- Makefile | 129 ++- README.md | 24 +- dev_tools/README.md | 7 +- dev_tools/requirements.txt | 14 - impatient-guide/Makefile | 166 ---- manual/Makefile | 29 - manual/source/Makefile | 48 -- manual/source/classes/Makefile | 35 - manual/source/classes/applications/Makefile | 59 -- manual/source/classes/base_classes/Makefile | 57 -- .../classes/contributed_definitions/Makefile | 59 -- requirements.txt | 13 +- dev_tools/setup.cfg => setup.cfg | 0 utils/build_preparation.py | 202 ----- utils/dev_nxdl2rst.py | 41 - utils/dev_units2rst.py | 13 - utils/local_utilities.py | 75 -- utils/nxdl2rst.py | 814 ------------------ utils/nxdl_desc2rst.py | 518 ----------- utils/nxdl_summary.py | 163 ---- utils/test_nxdl.py | 117 --- utils/test_nxdl2rst.py | 119 --- utils/test_suite.py | 32 - utils/types2rst.py | 36 - utils/units2rst.py | 105 --- utils/update_copyright_date.py | 1 - 31 files changed, 106 insertions(+), 3029 deletions(-) delete mode 100644 .github/workflows/publish-sphinx.yml delete mode 100644 .github/workflows/syntax-checks.yml delete mode 100644 dev_tools/requirements.txt delete mode 100644 impatient-guide/Makefile delete mode 100644 manual/Makefile delete mode 100755 manual/source/Makefile delete mode 100755 manual/source/classes/Makefile delete mode 100644 manual/source/classes/applications/Makefile delete mode 100644 manual/source/classes/base_classes/Makefile delete mode 100644 manual/source/classes/contributed_definitions/Makefile rename dev_tools/setup.cfg => setup.cfg (100%) delete mode 100644 utils/build_preparation.py delete mode 100755 utils/dev_nxdl2rst.py delete mode 100755 utils/dev_units2rst.py delete mode 100644 utils/local_utilities.py delete mode 100755 utils/nxdl2rst.py delete mode 100755 utils/nxdl_desc2rst.py delete mode 100755 utils/nxdl_summary.py delete mode 100644 utils/test_nxdl.py delete mode 100644 utils/test_nxdl2rst.py delete mode 100644 utils/test_suite.py delete mode 100755 utils/types2rst.py delete mode 100755 utils/units2rst.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 357145fc13..2187895160 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -43,19 +43,16 @@ jobs: - name: Install Requirements run: | python3 -m pip install --upgrade pip setuptools - python3 -m pip install -r dev_tools/requirements.txt + make install python3 -m pip list - name: Check Code Style run: | - cd dev_tools - black --check . - flake8 . - isort --check . + make style - name: Run Tests run: | - pytest dev_tools -v + make test - name: Install LaTeX run: | @@ -68,20 +65,20 @@ jobs: - name: Generate build files run: | - python3 -m dev_tools manual --prepare - python3 -m dev_tools impatient --prepare + make prepare - - name: Build PDF Manuals + - name: Build Impatient Guid run: | - sphinx-build -M latexpdf build/impatient-guide/ build/impatient-guide/build - cp build/impatient-guide/build/latex/NXImpatient.pdf build/manual/source/_static/NXImpatient.pdf - sphinx-build -M latexpdf build/manual/source/ build/manual/build - cp build/manual/build/latex/nexus.pdf build/manual/source/_static/NeXusManual.pdf + make impatient-guide + ls -lAFgh build/impatient-guide/build/html/index.html + ls -lAFgh build/impatient-guide/build/latex/NXImpatient.pdf - - name: Build HTML Manuals + - name: Build User Manual run: | - sphinx-build -b html build/manual/source/ build/manual/build/html - sphinx-build -b html -W build/impatient-guide/ build/impatient-guide/build/html + make pdf + make html + ls -lAFgh build/manual/build/html/index.html + ls -lAFgh build/manual/build/latex/nexus.pdf - name: Build and Commit the NeXus User Manual if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} diff --git a/.github/workflows/publish-sphinx.yml b/.github/workflows/publish-sphinx.yml deleted file mode 100644 index 7ae1b7fdf1..0000000000 --- a/.github/workflows/publish-sphinx.yml +++ /dev/null @@ -1,91 +0,0 @@ -name: Publish Sphinx Docs to GitHub Pages - -on: - # Triggers the workflow on push events but only for the main branch - push: - branches: - - main - tags: - - 'v2*' # all tags begining with v2 - -# see: https://sphinx-notes.github.io/pages/ -# see: https://github.com/marketplace/actions/sphinx-to-github-pages - -jobs: - - build: - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@master - with: - fetch-depth: 0 # otherwise, you will fail to push refs to dest repo - - - name: Install build requirements - run: | - pip install -r requirements.txt - - - name: Diagnostic - run: | - pip list - - - name: Run the test suite - run: | - make test - - - name: Prepare for out-of-source Sphinx build - run: | - make prepare - - - name: Diagnostic - run: | - ls -lAFGh - ls -lAFGh ./build - - - name: Install LaTeX - run: | - sudo apt-get update -y && \ - sudo apt-get install -y \ - latexmk \ - texlive-latex-recommended \ - texlive-latex-extra \ - texlive-fonts-recommended - - - name: Build impatient guide - run: | - make -C build impatient-guide - ls -lAFGh ./build/impatient-guide/_build/latex/*.pdf - - # Copy to documentation source directory - cp \ - ./build/impatient-guide/_build/latex/NXImpatient.pdf \ - ./build/manual/source/_static/ - - - name: Build PDF of manual - run: | - make -C build nxdl2rst pdf - - # Copy to documentation source directory - cp \ - ./build/manual/build/latex/nexus.pdf \ - ./build/manual/source/_static/NeXusManual.pdf - - - name: Include other PDFs - run: | - wget https://github.com/nexusformat/code/raw/master/doc/api/NeXusIntern.pdf -O ./build/manual/source/_static/NeXusIntern.pdf - wget https://github.com/nexusformat/code/raw/master/applications/NXtranslate/docs/NXtranslate.pdf -O ./build/manual/source/_static/NXtranslate.pdf - - - name: Build (html) and Commit - uses: sphinx-notes/pages@master - with: - # path to conf.py directory - documentation_path: build/manual/source - - - name: Publish if refs/tags - # remove/comment next line to push right away - if: startsWith(github.ref, 'refs/tags') - uses: ad-m/github-push-action@master - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - branch: gh-pages diff --git a/.github/workflows/syntax-checks.yml b/.github/workflows/syntax-checks.yml deleted file mode 100644 index 62cbf25f79..0000000000 --- a/.github/workflows/syntax-checks.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Syntax Checks - -on: - # Triggers the workflow on push or pull request events but only for the main branch - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build-linux: - name: CI py${{ matrix.python-version }} - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10'] - max-parallel: 5 - - steps: - - uses: actions/checkout@v2 - - - name: Install build requirements - run: | - pip install -r requirements.txt - - - name: Diagnostics - shell: bash -l {0} - run: | - pwd - ls -lart - - pip list - printenv | sort - - - name: Run tests - shell: bash -l {0} - run: | - make test - - - name: Build the documentation - shell: bash -l {0} - run: | - make prepare - make -C build nxdl2rst html - ls -lAGh diff --git a/.gitignore b/.gitignore index 5f82132a98..ff21c1627c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,18 +7,8 @@ __pycache__/ # Build artifacts build/ -_build/ makelog.txt -# Obsolete in-source build artifacts -manual/pdf -manual/source/nexus.pdf -manual/source/nxdl_desc.rst -manual/source/*.table -manual/source/classes/*/NX*.rst -manual/source/classes/*/*.nxdl.xml -manual/source/classes/contributed_definitions/canSAS - # Unknown /python/ __github_creds__.txt diff --git a/BUILDING.rst b/BUILDING.rst index 63e1b5aa86..6912ca1e6d 100644 --- a/BUILDING.rst +++ b/BUILDING.rst @@ -1,82 +1,4 @@ The documentation relies on Sphinx (a Python package) for its organization. The -GNU ``make`` program and Python are used to build the NeXus documentation. The -default build assembles the HTML version. You have the choice to build the -documentation in two places: +GNU ``make`` program and Python are used to build the NeXus documentation. The +default build assembles the HTML version. -* in the source tree -* out of the source tree - -Out-of-Tree documentation -========================= - -There are two ways to build out-of-source. - -Outside the source tree ------------------------ - -To build the NeXus documentation outside the -source tree, - -#. create the target directory for the documentation to be built:: - - mkdir /some/where/else - -#. note the definitions source directory - (the directory where this README file is located):: - - export SOURCE_DIR=/path/to/nexus/definitions - -#. copy the source to the target using this NeXus Python tool:: - - cd /some/where/else - python $(SOURCE_DIR)/utils/build_preparation.py $(SOURCE_DIR) - -#. build the documentation:: - - make clean - make - -The HTML documentation is located in this folder:: - - /some/where/else/manual/build/html/ - - -Inside the source tree, in a temporary directory ------------------------------------------------- - -Alternatively, as is a common practice with `cmake `_, -you can build *out-of-source* (sort of) in a temporary -``$(SOURCE_DIR)/build`` directory. For this, the *Makefile* -has the *builddir* rule:: - - export SOURCE_DIR=/path/to/nexus/definitions - cd $(SOURCE_DIR) - make builddir - cd build - make clean - make - -This is all done with one make command:: - - export SOURCE_DIR=/path/to/nexus/definitions - cd $(SOURCE_DIR) - make makebuilddir - -The HTML documentation is located in this folder:: - - $(SOURCE_DIR)/build/manual/build/html/ - -In-Tree documentation -===================== - -To build the NeXus documentation within the -source tree, go to the root directory -(the directory where this README file is located), -and type:: - - make clean - make - -The HTML documentation is located in this folder:: - - ./manual/build/html/ diff --git a/Makefile b/Makefile index c4bd30b99d..ae556d7339 100644 --- a/Makefile +++ b/Makefile @@ -3,109 +3,86 @@ # purpose: # build resources in NeXus definitions tree -SUBDIRS = manual impatient-guide PYTHON = python3 -DIR_NAME = "$(shell basename $(realpath .))" +SPHINX = sphinx-build BUILD_DIR = "build" -.PHONY: $(SUBDIRS) all clean html pdf nxdl2rst prepare test local help +.PHONY: help install style autoformat test clean prepare html pdf impatient-guide all local help :: @echo "" @echo "NeXus: Testing the NXDL files and building the documentation:" @echo "" - @echo "make all -C . Total (re)build of the manual, starting from root directory." - @echo "make all -C build Builds complete web site for the manual (in build directory)." - @echo "make clean Remove build products from some directories." - @echo "make impatient-guide -C build Build html & PDF versions of the Guide for the Impatient." - @echo "make html -C build Build HTML version of manual, requires nxdl2rst first," - @echo "make nxdl2rst -C build Document each NXDL class, add to manual source." - @echo "make pdf -C build Build PDF version of manual, requires nxdl2rst first" - @echo "make prepare -C . (Re)create the build directory." - @echo "make test Apply all Python-coded tests." - @echo "make local -C . (Developer use) Build nxdl2rst and the html manual." + + @echo "make install Install all requirements to run tests and builds." + @echo "make style Check python coding style." + @echo "make autoformat Format all files to the coding style conventions." + @echo "make test Run NXDL syntax and documentation tests." + @echo "make clean Remove all build files." + @echo "make prepare (Re)create all build files." + @echo "make html Build HTML version of manual. Requires prepare first." + @echo "make pdf Build PDF version of manual. Requires prepare first." + @echo "make impatient-guide Build html & PDF versions of the Guide for the Impatient. Requires prepare first." + @echo "make all Builds complete web site for the manual (in build directory)." + @echo "make local (Developer use) Test, prepare and build the HTML manual." @echo "" @echo "Note: All builds of the manual will occur in the 'build/' directory." @echo " For a complete build, run 'make all' in the root directory." - @echo " To delete and then create the 'build/' directory, run 'make prepare' in the root directory." - @echo " For syntax checks of the NXDL files, run 'make test' in either root or 'build/' directory." - @echo " Developers might find it easier to 'make local' to confirm the documentation builds." + @echo " Developers of the NeXus class definitions can use 'make local' to" + @echo " confirm the documentation builds." @echo "" -all :: -ifneq ($(DIR_NAME), $(BUILD_DIR)) - # root directory - $(MAKE) test - $(MAKE) prepare - $(MAKE) -C $(BUILD_DIR) all -else - # root/build directory - $(MAKE) clean - $(MAKE) impatient-guide - $(MAKE) nxdl2rst - $(MAKE) html - $(MAKE) pdf +install :: + $(PYTHON) -m pip install -r requirements.txt - cp impatient-guide/_build/latex/NXImpatient.pdf manual/build/html/_static/NXImpatient.pdf - cp manual/build/latex/nexus.pdf manual/build/html/_static/NeXusManual.pdf +style :: + $(PYTHON) -m black --check dev_tools + $(PYTHON) -m flake8 dev_tools + $(PYTHON) -m isort --check dev_tools - @echo "HTML built: `ls -lAFgh manual/build/html/index.html`" - @echo "PDF built: `ls -lAFgh manual/build/latex/nexus.pdf`" -endif +autoformat :: + $(PYTHON) -m black dev_tools + $(PYTHON) -m isort dev_tools -clean :: - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done - -impatient-guide :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) html -C $@ - $(MAKE) latexpdf -C $@ -endif +test :: + $(PYTHON) -m pytest dev_tools -html :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) html -C manual -endif +clean :: + $(RM) -rf $(BUILD_DIR) -nxdl2rst :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - $(MAKE) -C manual/source PYTHON=$(PYTHON) -endif +prepare :: + $(PYTHON) -m dev_tools manual --prepare --build-root $(BUILD_DIR) + $(PYTHON) -m dev_tools impatient --prepare --build-root $(BUILD_DIR) pdf :: -ifeq ($(DIR_NAME), $(BUILD_DIR)) - # expect pass to fail (thus exit 0) since nexus.ind not found first time - # extra option needed to satisfy "levels nested too deeply" error - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) - - # create the missing file - makeindex -s manual/build/latex/python.ist manual/build/latex/nexus.idx + $(SPHINX) -M latexpdf $(BUILD_DIR)/manual/source/ $(BUILD_DIR)/manual/build + cp $(BUILD_DIR)/manual/build/latex/nexus.pdf $(BUILD_DIR)/manual/source/_static/NeXusManual.pdf - # second pass will also fail but we can ignore it without problem - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) - - # third time should be no errors - ($(MAKE) latexpdf LATEXOPTS="--interaction=nonstopmode -f" -C manual || exit 0) -endif - -prepare :: -ifneq ($(DIR_NAME), $(BUILD_DIR)) - $(RM) -rf $(BUILD_DIR) - mkdir -p $(BUILD_DIR) - $(PYTHON) utils/build_preparation.py . $(BUILD_DIR) -endif +html :: + $(SPHINX) -b html -W $(BUILD_DIR)/manual/source/ $(BUILD_DIR)/manual/build/html -test :: - $(PYTHON) utils/test_suite.py +impatient-guide :: + $(SPHINX) -b html -W $(BUILD_DIR)/impatient-guide/ $(BUILD_DIR)/impatient-guide/build/html + $(SPHINX) -M latexpdf $(BUILD_DIR)/impatient-guide/ $(BUILD_DIR)/impatient-guide/build + cp $(BUILD_DIR)/impatient-guide/build/latex/NXImpatient.pdf $(BUILD_DIR)/manual/source/_static/NXImpatient.pdf # for developer's use on local build host local :: $(MAKE) test $(MAKE) prepare - $(MAKE) nxdl2rst -C $(BUILD_DIR) - $(MAKE) html -C $(BUILD_DIR) + $(MAKE) html + +all :: + $(MAKE) clean + $(MAKE) prepare + $(MAKE) impatient-guide + $(MAKE) pdf + $(MAKE) html + @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/impatient-guide/build/html/index.html`" + @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/impatient-guide/build/latex/NXImpatient.pdf`" + @echo "HTML built: `ls -lAFgh $(BUILD_DIR)/manual/build/html/index.html`" + @echo "PDF built: `ls -lAFgh $(BUILD_DIR)/manual/build/latex/nexus.pdf`" + # NeXus - Neutron and X-ray Common Data Format # diff --git a/README.md b/README.md index 2758ea621c..a9e2e3ac42 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,33 @@ * License: [![License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) * Test, Build and Deploy: [![Workflow](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml) +## NeXus definition developers + +After make a change to the NeXus class definitions there are two important steps +to take before commiting the change: + + 1. check whether the change do not violate any syntax rules + 2. verify whether the change looks as intended in the HTML documentation + +First install the test and build requirements with this command (only run once) + + make install + +Then verify syntax and build the HTML manual with this command + + make local + +Open the HTML manual in a web brower for visual verification + + firefox build/manual/build/html/index.html + +## Repository content + These are the components that define the structure of NeXus data files in the development directory. component | description -------------------------------|------------------------ -[BUILDING.rst](BUILDING.rst) | how to build the documentation [CHANGES.rst](CHANGES.rst) | Change History [LGPL.txt](LGPL.txt) | one proposed license model [NXDL_VERSION](NXDL_VERSION) | the current NXDL version number @@ -28,3 +49,4 @@ package/ | directory for packaging this content utils/ | various tools used in the definitions tree www/ | launch (home) page of NeXus WWW site xslt/ | various XML stylesheet transformations +dev_tools/ | developer tools for testing and building \ No newline at end of file diff --git a/dev_tools/README.md b/dev_tools/README.md index 4f42b489e1..efdc54bd3d 100644 --- a/dev_tools/README.md +++ b/dev_tools/README.md @@ -44,10 +44,9 @@ sphinx-build -b html -W build/manual/source/ build/manual/build/html Auto-formatting and syntax checking to the developer tools themselves ```bash -cd dev_tools -black . -isort . -flake8 . +black dev_tools +isort dev_tools +flake8 dev_tools ``` ## Prepare environment (optional) diff --git a/dev_tools/requirements.txt b/dev_tools/requirements.txt deleted file mode 100644 index 815406521c..0000000000 --- a/dev_tools/requirements.txt +++ /dev/null @@ -1,14 +0,0 @@ -# Prepare for Documentation -lxml -pyyaml - -# Documentation building -sphinx>=5 - -# Testing -pytest - -# Code style and auto-formatting -black>=22.3 -flake8>=4 -isort>=5.10 \ No newline at end of file diff --git a/impatient-guide/Makefile b/impatient-guide/Makefile deleted file mode 100644 index fa3c25dcf6..0000000000 --- a/impatient-guide/Makefile +++ /dev/null @@ -1,166 +0,0 @@ -# Makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = _build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext - -help: - @echo "Please use \`make ' where is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - touch $(BUILDDIR)/html/.nojekyll - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/NeXusfortheImpatient.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/NeXusfortheImpatient.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/NeXusfortheImpatient" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/NeXusfortheImpatient" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." - -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." - -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." - -PDF_version: latexpdf - cp $(BUILDDIR)/latex/NXImpatient.pdf ./ - -EPUB_version: epub - cp $(BUILDDIR)/epub/NeXusfortheImpatient.epub ./NXImpatient.epub - -HTML_version:: PDF_version html - -all: PDF_version HTML_version - -rebuild: clean all diff --git a/manual/Makefile b/manual/Makefile deleted file mode 100644 index 06a3752a77..0000000000 --- a/manual/Makefile +++ /dev/null @@ -1,29 +0,0 @@ -# Minimal makefile for Sphinx documentation -# - -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -SPHINXPROJ = NeXusManual -SOURCEDIR = source -BUILDDIR = build - -# Put it first so that "make" without argument is like "make help". -help: - @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -.PHONY: help Makefile - -# # TODO: where to add this in next line -# # https://github.com/nexusformat/definitions/issues/659 -# # LATEXOPTS="--interaction=nonstopmode" -# latexpdf :: Makefile -# @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -# Catch-all target: route all unknown targets to Sphinx using the new -# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). -%: Makefile - @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) - -clean: - $(MAKE) clean -C $(SOURCEDIR) diff --git a/manual/source/Makefile b/manual/source/Makefile deleted file mode 100755 index da138be897..0000000000 --- a/manual/source/Makefile +++ /dev/null @@ -1,48 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - - -all :: types.table units.table nxdl_desc.rst subdirs - -types.table: ../../utils/types2rst.py ../../utils/units2rst.py ../../nxdlTypes.xsd - $(PYTHON) -u ../../utils/types2rst.py ../../nxdlTypes.xsd > $@ - -units.table: ../../utils/units2rst.py ../../nxdlTypes.xsd - $(PYTHON) -u ../../utils/units2rst.py ../../nxdlTypes.xsd > $@ - -nxdl_desc.rst: ../../utils/nxdl_desc2rst.py ../../nxdl.xsd - $(PYTHON) -u ../../utils/nxdl_desc2rst.py ../../nxdl.xsd > $@ - -SUBDIRS = classes - -.PHONY: subdirs $(SUBDIRS) - -subdirs: $(SUBDIRS) - -$(SUBDIRS): - $(MAKE) -C $@ - -clean: - $(RM) types.table units.table nxdl_desc.rst - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done diff --git a/manual/source/classes/Makefile b/manual/source/classes/Makefile deleted file mode 100755 index be5f8bf8fc..0000000000 --- a/manual/source/classes/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -SUBDIRS = base_classes applications contributed_definitions - -.PHONY: subdirs $(SUBDIRS) clean - -subdirs: $(SUBDIRS) - -$(SUBDIRS): - $(MAKE) -C $@ - -clean: - for dir in $(SUBDIRS); do \ - $(MAKE) -C $$dir clean; \ - done diff --git a/manual/source/classes/applications/Makefile b/manual/source/classes/applications/Makefile deleted file mode 100644 index 2d6bc33d2e..0000000000 --- a/manual/source/classes/applications/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/applications -SUBDIRS = canSAS - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to applications/index.rst" - $(PYTHON) $(NXDLSUMMARY) applications - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - $(RM) -rf $(SUBDIRS) - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/manual/source/classes/base_classes/Makefile b/manual/source/classes/base_classes/Makefile deleted file mode 100644 index 3b124728c0..0000000000 --- a/manual/source/classes/base_classes/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/base_classes - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to base_classes/index.rst" - $(PYTHON) $(NXDLSUMMARY) base_classes - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/manual/source/classes/contributed_definitions/Makefile b/manual/source/classes/contributed_definitions/Makefile deleted file mode 100644 index 6ddbadb18c..0000000000 --- a/manual/source/classes/contributed_definitions/Makefile +++ /dev/null @@ -1,59 +0,0 @@ -# Makefile for NeXus manual custom pages - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org - -ROOTDIR = ../../../.. -NXDL2RST = $(ROOTDIR)/utils/nxdl2rst.py -NXDLSUMMARY= $(ROOTDIR)/utils/nxdl_summary.py -NXDL_SUFFIX = nxdl.xml - -SRCDIR = $(ROOTDIR)/contributed_definitions -SUBDIRS = container - -SRCs = $(wildcard $(SRCDIR)/NX*.$(NXDL_SUFFIX) ) -RSTs := $(SRCs:.$(NXDL_SUFFIX)=.rst) -RSTs := $(patsubst $(SRCDIR)/%, ./%, $(RSTs) ) -TARGET_NXDLs = $(patsubst $(SRCDIR)/%, ./%, $(SRCs) ) - -#vpath %.$(NXDL_SUFFIX) $(SRCDIR) - -.PHONY: all clean test - -all :: index.rst $(RSTs) $(TARGET_NXDLs) - -%.nxdl.xml : $(SRCDIR)/%.$(NXDL_SUFFIX) Makefile - cp $(SRCDIR)/$@ ./$@ - -index.rst :: - echo "Adding summaries to contributed_definitions/index.rst" - $(PYTHON) $(NXDLSUMMARY) contributed_definitions - -%.rst : %.$(NXDL_SUFFIX) $(NXDL2RST) Makefile - $(PYTHON) -u $(NXDL2RST) $< > $@ - -clean :: - $(RM) index.rst NX*.rst NX*.nxdl.xml - $(RM) -rf $(SUBDIRS) - -test :: - # $(SRCs) - # ----------- - # $(RSTs) diff --git a/requirements.txt b/requirements.txt index 070e65c74f..815406521c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,14 @@ +# Prepare for Documentation lxml pyyaml -Sphinx + +# Documentation building +sphinx>=5 + +# Testing +pytest + +# Code style and auto-formatting +black>=22.3 +flake8>=4 +isort>=5.10 \ No newline at end of file diff --git a/dev_tools/setup.cfg b/setup.cfg similarity index 100% rename from dev_tools/setup.cfg rename to setup.cfg diff --git a/utils/build_preparation.py b/utils/build_preparation.py deleted file mode 100644 index ecc5876e0e..0000000000 --- a/utils/build_preparation.py +++ /dev/null @@ -1,202 +0,0 @@ -#!/usr/bin/env python - -""" -Copy all resources for out-of-source documentation build - -Since we provide a build for Linux, MacOSX, and Windows, -this tool must be multiplatform. If only for Linux -(and possibly MacOSX), it might be possible to use a form of:: - - cp -a ../base_classes ./ - cp -a ../applications ./ - cp -a ../contributed_definitions ./ - cp -a ../manual ./ - ... - -Here, we identify and copy all resources to build. -The target directory is assumed to be the current directory. - -""" - -# Re-run this code to bring in any changed files (for incremental build) -# Be sure to properly specify the source and target directories. - -import os, sys -from local_utilities import replicate - - -MTIME_TOLERANCE = 0.001 # ignore mtime differences <= 1 ms -ROOT_DIR_EXPECTED_RESOURCES = { - "files": """COPYING LGPL.txt Makefile NXDL_VERSION - nxdl.xsd nxdlTypes.xsd README.md - """.split(), - "subdirs": """applications base_classes contributed_definitions manual - package utils www impatient-guide - """.split(), -} -REPLICATED_RESOURCES = """ - LGPL.txt Makefile nxdl.xsd nxdlTypes.xsd NXDL_VERSION - base_classes applications contributed_definitions - manual utils impatient-guide -""".split() - - -def mtime_size(filename): - """get the modification time and size of the given item""" - file_status = os.stat(filename) - return file_status.st_mtime, file_status.st_size - - -def standardize_name(path, resource_name): - """always use the absolute path to the filesystem resource""" - return os.path.abspath(os.path.join(path, resource_name)) - - -def identical(source, target): - """compare if the resource is the same on both paths""" - if not os.path.exists(target): - return False - s_mtime, s_size = mtime_size(source) - t_mtime, t_size = mtime_size(target) - return abs(s_mtime - t_mtime) <= MTIME_TOLERANCE and s_size == t_size - - -def get_source_items(resources, source_path): - """walk the source_path directories accumulating files to be checked""" - file_list = [] - path_list = [] - for path in sorted(resources): - source = standardize_name(source_path, path) - if os.path.isfile(source): - file_list.append(source) - else: - for root, dirs, files in os.walk(source): - path_list.append(root) - file_list = file_list + [os.path.join(root, _) for _ in files] - return path_list, file_list - - -def is_definitions_directory(basedir): - """test if ``basedir`` is a NeXus definitions directory""" - # look for the expected files and subdirectories in the root directory - for item_list in ROOT_DIR_EXPECTED_RESOURCES.values(): - for item in item_list: - if not os.path.exists(os.path.join(basedir, item)): - return False - return True - - -def qualify_inputs(source_dir, target_path): - """raise error if this program cannot continue, based on the inputs""" - if not os.path.exists(source_dir): - raise RuntimeError("Cannot find " + source_dir) - - if not os.path.isdir(source_dir): - raise RuntimeError("Not a directory: " + source_dir) - - if not is_definitions_directory(source_dir): - msg = "Not a NeXus definitions root directory " + source_dir - raise RuntimeError(msg) - - if source_dir == target_path: - msg = "Source and target directories cannot be the same" - raise RuntimeError(msg) - - -def command_args(): - """get the command-line arguments, handle syntax errors""" - import argparse - - doc = __doc__.strip().splitlines()[0] - parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "defs_dir", action="store", help="path to NeXus definitions root directory" - ) - parser.add_argument( - "build_dir", - action="store", - default=None, - nargs="?", - help="path to target directory (default: current directory)", - ) - return parser.parse_args() - - -def update(source_path, target_path): - """ - duplicate directory from source_path to target_path - - :param source_path str: source directory (NeXus definitions dir) - :param target_path str: target directory is specified for build product - """ - # TODO: what about file items in target_path that are not in source_path? - source_path = os.path.abspath(source_path) - target_path = os.path.abspath(target_path) - qualify_inputs(source_path, target_path) - - paths, files = get_source_items(REPLICATED_RESOURCES, source_path) - print("source has %d directories and %d files" % (len(paths), len(files))) - - # create all the directories / subdirectories - for source in sorted(paths): - relative_name = source[len(source_path) :].lstrip(os.sep) - target = standardize_name(target_path, relative_name) - if not os.path.exists(target): - print("create directory %s" % target) - os.mkdir(target, os.stat(source_path).st_mode) - # check if the files need to be updated - for source in sorted(files): - relative_name = source[len(source_path) :].lstrip(os.sep) - target = standardize_name(target_path, relative_name) - if not identical(source, target): - print("update file %s" % target) - replicate(source, target) - - -def main(): - """ - standard command-line processing - - source directory (NeXus definitions dir) named as command line argument - target directory is specified (or defaults to present working directory) - """ - cli = command_args() - source_path = os.path.abspath(cli.defs_dir) - target_path = cli.build_dir or os.path.abspath(os.getcwd()) - update(source_path, target_path) - - -def __developer_build_setup__(): - """for use with source-code debugger ONLY""" - import shutil - - # sys.argv.append('-h') - os.chdir("../") - os.chdir("build") - sys.argv.append("..") - - -if __name__ == "__main__": - # __developer_build_setup__() - main() - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2015 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/dev_nxdl2rst.py b/utils/dev_nxdl2rst.py deleted file mode 100755 index 6bbe3e2bf1..0000000000 --- a/utils/dev_nxdl2rst.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python - -""" -Developers: use this code to develop and test nxdl2rst.py -""" - -# testing: -# cd /tmp -# mkdir out -# /G/nx-def/utils/nxdl2rst.py /G/nx-def/applications/NXsas.nxdl.xml > nxsas.rst && sphinx-build . out -# then point browser to file:///tmp/out/nxsas.html - - -import nxdl2rst -import os -import sys - - -# find the directory of this python file -BASEDIR = os.path.dirname(__file__) - - -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXarchive.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXcanSAS.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXmx.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'applications', 'NXsas.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXcrystal.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXentry.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXobject.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXroot.nxdl.xml') -# nxdl = os.path.join(BASEDIR, '..', 'base_classes', 'NXuser.nxdl.xml') -nxdl = os.path.join(BASEDIR, "..", "applications", "NXarpes.nxdl.xml") -# nxdl = os.path.join(BASEDIR, '..', 'contributed_definitions', 'NXmagnetic_kicker.nxdl.xml') - - -if len(sys.argv) == 1: - sys.argv.append(nxdl) -elif len(sys.argv) > 1: - sys.argv[1] = nxdl - -nxdl2rst.main() diff --git a/utils/dev_units2rst.py b/utils/dev_units2rst.py deleted file mode 100755 index b43d05f29a..0000000000 --- a/utils/dev_units2rst.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python - -""" -Developers: use this code to develop and test nxdl2rst.py -""" - -import sys -from units2rst import worker - - -# sys.argv.append("../nxdlTypes.xsd") -sys.argv.append("nxdlTypes.xsd") -worker("anyUnitsAttr") diff --git a/utils/local_utilities.py b/utils/local_utilities.py deleted file mode 100644 index 54a3db6313..0000000000 --- a/utils/local_utilities.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python - -""" -Common code for NeXus definitions Python tools - -====================== ================================== -tool description -====================== ================================== -:meth:`printf` formatted print without newline -:meth:`mtime` return file modification time -:meth:`replicate` copy directory stack or file -:meth:`replicate_tree` copy directory stack -====================== ================================== - -""" - -import os -import shutil - - -def mtime(file_name): - """return file modification time""" - return os.stat(file_name)[os.stat.ST_MTIME] - - -def replicate(source, target): - """ - for directories or files: copy ``source`` to ``target``, replaces ``target`` - - :param str source: path to source resource - :param str target: path to target location - """ - if os.path.isfile(source): - shutil.copy2(source, target) - elif os.path.isdir(source): - replicate_tree(source, target) - else: - msg = "Do not know how to copy (skipped): " + source - raise RuntimeWarning(msg) - - -def replicate_tree(source, target): - """ - for directories: copy ``source`` to ``target``, replaces ``target`` - - :param str source: path to source resource (a directory) - :param str target: path to target location (a directory) - """ - if os.path.exists(source): - if os.path.exists(target): - shutil.rmtree(target, ignore_errors=True) - shutil.copytree(source, target) - else: - raise RuntimeError("Directory not found: " + source) - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/nxdl2rst.py b/utils/nxdl2rst.py deleted file mode 100755 index 2493046525..0000000000 --- a/utils/nxdl2rst.py +++ /dev/null @@ -1,814 +0,0 @@ -#!/usr/bin/env python - -""" -Read the NeXus NXDL class specification and describe it. -Write a restructured text (.rst) document for use in the NeXus manual in -the NeXus NXDL Classes chapter. -""" - -# testing: see file dev_nxdl2rst.py - -from collections import OrderedDict -from html import parser as HTMLParser -import datetime -import json -import lxml.etree -import os -import pathlib -import re -import sys -import yaml -from local_utilities import replicate - - -INDENTATION_UNIT = " " -use_application_defaults = None -repo_root_path = pathlib.Path(__file__).parent.parent -WRITE_ANCHOR_REGISTRY = False -HTML_ROOT = "https://github.com/nexusformat/definitions/blob/main" -MANUAL_ROOT = "https://manual.nexusformat.org/" - - -class AnchorRegistry: - """Document the NXDL vocabulary.""" - - def __init__(self) -> None: - path = repo_root_path / "manual" / "source" / "_static" - base = "nxdl_vocabulary" - self.html_file = path / f"{base}.html" - self.txt_file = path / f"{base}.txt" - self.json_file = path / f"{base}.json" - self.yaml_file = path / f"{base}.yml" - self.registry = self._read() - self.local_anchors = [] # anchors from current NXDL file - self.nxdl_file = None - self.nxdl_subdir = None - - @property - def all_anchors(self): - result = [] - for v in self.registry.values(): - result += list(v.keys()) - return result - - def add(self, anchor): - if anchor not in self.local_anchors: - self.local_anchors.append(anchor) - - key = self.key_from_anchor(anchor) - - if key not in self.registry: - self.registry[key] = {} - - reg = self.registry[key] - if anchor not in reg: - hanchor = self._html_anchor(anchor) - fnxdl = "/".join(pathlib.Path(self.nxdl_file).parts[-2:]).split(".")[0] - url = f"{MANUAL_ROOT}classes/{self.nxdl_subdir}/{fnxdl}.html{hanchor}" - reg[anchor] = dict(term=anchor, html=hanchor, url=url,) - - def key_from_anchor(self, anchor): - key = anchor.lower().split("/")[-1].split("@")[-1].split("-")[0] - if "@" in anchor: - # restore preceding "@" symbol - key = "@" + key - return key - - def write(self): - # fmt: off - version = open(repo_root_path / "NXDL_VERSION", "r").read().strip() - contents = dict( - _metadata=dict( - datetime=datetime.datetime.utcnow().isoformat(), - title="NeXus NXDL vocabulary.", - subtitle=( - "Anchors for all NeXus fields," - " groups, attributes, and links." - ), - version=version, - ), - terms=self.registry, - ) - # fmt: on - - self._write_yaml(contents) - self._write_json(contents) - self._write_txt() - self._write_html(contents) - - def _html_anchor(self, anchor): - """ - Create (internal hyperlink target for) HTML anchor from reST anchor. - - Example: - - * reST anchor: /NXcanSAS/ENTRY/TRANSMISSION_SPECTRUM@timestamp-attribute - * HTML anchor: #nxcansas-entry-transmission-spectrum-timestamp-attribute - """ - html_anchor = ( - anchor.lower() - .lstrip("/") - .replace("_", "-") - .replace("@", "-") - .replace("/", "-") - ) - return f"#{html_anchor}" - - def _read(self): - """The YAML file will record anchors (terms) from all NXDL files.""" - registry = None - if self.yaml_file.exists(): - contents = yaml.load(open(self.yaml_file, "r").read(), Loader=yaml.Loader) - if contents is not None: - registry = contents.get("terms") - return registry or {} - - def _write_html(self, contents): - """Write the anchors to an HTML file.""" - root = lxml.etree.Element("html") - body = lxml.etree.SubElement(root, "body") - title = lxml.etree.SubElement(body, "h1") - subtitle = lxml.etree.SubElement(body, "em") - - title.text = contents["_metadata"]["title"].strip(".") - subtitle.text = contents["_metadata"]["subtitle"].strip(".") - vocab_list = lxml.etree.SubElement(body, "h2") - vocab_list.text = "NXDL Vocabulary" - - p = lxml.etree.SubElement(body, "p") - p.text = "This content is also available in these formats: " - for ext in "json txt yml".split(): - a = lxml.etree.SubElement(p, "a") - a.attrib["href"] = f"{MANUAL_ROOT}_static/{self.txt_file.stem}.{ext}" - a.text = f" {ext}" - - dl = lxml.etree.SubElement(body, "dl") - for term, termlist in sorted(contents["terms"].items()): - dterm = lxml.etree.SubElement(dl, "dt") - dterm.text = term - for _, itemdict in sorted(termlist.items()): - ddef = lxml.etree.SubElement(dterm, "dd") - a = lxml.etree.SubElement(ddef, "a") - a.attrib["href"] = itemdict["url"] - a.text = itemdict["term"] - - lxml.etree.SubElement(body, "hr") - - foot = lxml.etree.SubElement(body, "p") - foot_em = lxml.etree.SubElement(foot, "em") - foot_em.text = f"written: {contents['_metadata']['datetime']}" - - html = lxml.etree.tostring(root, pretty_print=True).decode() - with open(self.html_file, "w") as f: - f.write(html) - f.write("\n") - - def _write_json(self, contents): - with open(self.json_file, "w") as f: - json.dump(contents, f, indent=4, sort_keys=True) - f.write("\n") - - def _write_txt(self): - """Compendium (dump the list of all known anchors in raw form).""" - terms = self.all_anchors - with open(self.txt_file, "w") as f: - f.write("\n".join(sorted(terms))) - f.write("\n") - - def _write_yaml(self, contents): - with open(self.yaml_file, "w") as f: - yaml.dump(contents, f) - - -anchor_registry = AnchorRegistry() - - -def printAnchorList(): - """Print the list of hypertext anchors.""" - - def sorter(key): - return key.lower() - - if len(anchor_registry.local_anchors) > 0: - if WRITE_ANCHOR_REGISTRY: - # ONLY in the build directory - anchor_registry.write() - - print("") - print("Hypertext Anchors") - print("-----------------\n") - print( - "List of hypertext anchors for all groups, fields,\n" - "attributes, and links defined in this class.\n\n" - ) - # fmt: off - rst = [ - f"* :ref:`{ref} <{ref}>`" - for ref in sorted(anchor_registry.local_anchors, key=sorter) - ] - # fmt: on - print("\n".join(rst)) - - -def fmtTyp(node): - typ = node.get("type", ":ref:`NX_CHAR `") # per default - if typ.startswith("NX_"): - typ = ":ref:`%s <%s>`" % (typ, typ) - return typ - - -def fmtUnits(node): - units = node.get("units", "") - if not units: - return "" - if units.startswith("NX_"): - units = "\ :ref:`%s <%s>`" % (units, units) - return " {units=%s}" % units - - -def getDocBlocks(ns, node): - docnodes = node.xpath("nx:doc", namespaces=ns) - if docnodes is None or len(docnodes) == 0: - return "" - if len(docnodes) > 1: - raise Exception( - "Too many doc elements: line %d, %s" - % (node.sourceline, os.path.split(node.base)[1]) - ) - docnode = docnodes[0] - - # be sure to grab _all_ content in the documentation - # it might look like XML - s = lxml.etree.tostring( - docnode, pretty_print=True, method="c14n", with_comments=False - ).decode("utf-8") - m = re.search(r"^]*>\n?(.*)\n?$", s, re.DOTALL) - if not m: - raise Exception("unexpected docstring [%s] " % s) - text = m.group(1) - - # substitute HTML entities in markup: "<" for "<" - # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string - htmlparser = HTMLParser.HTMLParser() - try: # see #661 - import html - - text = html.unescape(text) - except (ImportError, AttributeError): - text = htmlparser.unescape(text) - - # Blocks are separated by whitelines - blocks = re.split("\n\s*\n", text) - if len(blocks) == 1 and len(blocks[0].splitlines()) == 1: - return [blocks[0].rstrip().lstrip()] - - # Indentation must be given by first line - m = re.match(r"(\s*)(\S+)", blocks[0]) - if not m: - return [""] - indent = m.group(1) - - # Remove common indentation as determined from first line - if indent == "": - raise Exception( - "Missing initial indentation in of %s [%s]" - % (node.get("name"), blocks[0]) - ) - - out_blocks = [] - for block in blocks: - lines = block.rstrip().splitlines() - out_lines = [] - for line in lines: - if line[: len(indent)] != indent: - raise Exception( - 'Bad indentation in of %s [%s]: expected "%s" found "%s".' - % ( - node.get("name"), - block, - re.sub(r"\t", "\\\\t", indent), - re.sub(r"\t", "\\\\t", line), - ) - ) - out_lines.append(line[len(indent) :]) - out_blocks.append("\n".join(out_lines)) - return out_blocks - - -def getDocLine(ns, node): - blocks = getDocBlocks(ns, node) - if len(blocks) == 0: - return "" - if len(blocks) > 1: - raise Exception("Unexpected multi-paragraph doc [%s]" % "|".join(blocks)) - return re.sub(r"\n", " ", blocks[0]) - - -def get_minOccurs(node): - """ - get the value for the ``minOccurs`` attribute - - :param obj node: instance of lxml.etree._Element - :returns str: value of the attribute (or its default) - """ - # TODO: can we improve on the default by examining nxdl.xsd? - minOccurs_default = {True: "1", False: "0"}[use_application_defaults] - minOccurs = node.get("minOccurs", minOccurs_default) - return minOccurs - - -def get_required_or_optional_text(node): - """ - make clear if a reported item is required or optional - - :param obj node: instance of lxml.etree._Element - :returns: formatted text - """ - tag = node.tag.split("}")[-1] - nm = node.get("name") - if tag in ("field", "group"): - optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) - minOccurs = get_minOccurs(node) - if recommended: - optional_text = "(recommended) " - elif minOccurs in ("0", 0) or optional: - optional_text = "(optional) " - elif minOccurs in ("1", 1): - optional_text = "(required) " - else: - # this is unexpected and remarkable - # TODO: add a remark to the log - optional_text = "(``minOccurs=%s``) " % str(minOccurs) - elif tag in ("attribute",): - optional_default = not use_application_defaults - optional = node.get("optional", optional_default) in (True, "true", "1", 1) - recommended = node.get("recommended", None) in (True, "true", "1", 1) - optional_text = {True: "(optional) ", False: "(required) "}[optional] - if recommended: - optional_text = "(recommended) " - else: - optional_text = "(unknown tag: " + str(tag) + ") " - return optional_text - - -def analyzeDimensions(ns, parent): - """These are the different dimensions that can occur: - - 1. Fixed rank - - - - - - - - 2. Variable rank because of optional dimensions - - - - - - - - - 3. Variable rank because no dimensions specified - - - - - The legacy way of doing this (still supported) - - - - - - 4. Rank and dimensions equal to that of another field called `field_name` - - - - - """ - node_list = parent.xpath("nx:dimensions", namespaces=ns) - if len(node_list) != 1: - return "" - node = node_list[0] - node_list = node.xpath("nx:dim", namespaces=ns) - - dims = [] - optional = False - for subnode in node_list: - # Dimension index (starts from index 1) - index = subnode.get("index", "") - if not index.isdigit(): - raise RuntimeError("A dimension must have an index") - index = int(index) - if index == 0: - # No longer needed: legacy way to specify that the - # rank is variable - continue - - # Expand dimensions when needed - index -= 1 - nadd = max(index - len(dims) + 1, 0) - if nadd: - dims += ["."] * nadd - - # Dimension symbol - dim = subnode.get("value") # integer or symbol from the table - if not dim: - ref = subnode.get("ref") - if ref: - return " (Rank: same as field %s, Dimensions: same as field %s)" % ( - ref, - ref, - ) - dim = "." # dimension has no symbol - - # Dimension might be optional - if subnode.get("required", "true").lower() == "false": - optional = True - elif optional: - raise RuntimeError( - "A required dimension cannot come after an optional dimension" - ) - if optional: - dim = "[%s]" % dim - - dims[index] = dim - - # When the rank is missing, set to the number of dimensions when - # there are dimensions specified and none of them are optional. - ndims = len(dims) - rank = node.get("rank", None) - if rank is None and not optional and ndims: - rank = str(ndims) - - # Validate rank and dimensions - rank_is_fixed = rank and rank.isdigit() - if optional and rank_is_fixed: - raise RuntimeError("A fixed rank cannot have optional dimensions") - if rank_is_fixed and ndims and int(rank) != ndims: - raise RuntimeError("The rank and the number of dimensions do not correspond") - - # Omit rank and/or dimensions when not specified - if rank and dims: - dims = ", ".join(dims) - return " (Rank: %s, Dimensions: [%s])" % (rank, dims) - elif rank: - return " (Rank: %s)" % rank - elif dims: - dims = ", ".join(dims) - return " (Dimensions: [%s])" % dims - return "" - - -def hyperlinkTarget(parent_path, name, nxtype): - """Return internal hyperlink target for HTML anchor.""" - if nxtype == "attribute": - sep = "@" - else: - sep = "/" - target = "%s%s%s-%s" % (parent_path, sep, name, nxtype) - anchor_registry.add(target) - return ".. _%s:\n" % target - - -def printEnumeration(indent, ns, parent): - node_list = parent.xpath("nx:item", namespaces=ns) - if len(node_list) == 0: - return "" - - if len(node_list) == 1: - print(f"{indent}Obligatory value:", end="") - else: - print(f"{indent}Any of these values:", end="") - - docs = OrderedDict() - for item in node_list: - name = item.get("value") - docs[name] = getDocLine(ns, item) - - ENUMERATION_INLINE_LENGTH = 60 - - def show_as_typed_text(msg): - return "``%s``" % msg - - oneliner = " | ".join(map(show_as_typed_text, docs.keys())) - if any(doc for doc in docs.values()) or len(oneliner) > ENUMERATION_INLINE_LENGTH: - # print one item per line - print("\n") - for name, doc in docs.items(): - print(f"{indent} * {show_as_typed_text(name)}", end="") - if doc: - print(f": {doc}", end="") - print("\n") - else: - # print all items in one line - print(f" {oneliner}") - print("") - - -def printDoc(indent, ns, node, required=False): - blocks = getDocBlocks(ns, node) - if len(blocks) == 0: - if required: - raise Exception("No documentation for: " + node.get("name")) - print("") - else: - for block in blocks: - for line in block.splitlines(): - print(f"{indent}{line}") - print() - - -def printAttribute(ns, kind, node, optional, indent, parent_path): - name = node.get("name") - index_name = name - print(f"{indent}" f"{hyperlinkTarget(parent_path, name, 'attribute')}") - print(f"{indent}.. index:: {index_name} ({kind} attribute)\n") - print(f"{indent}**@{name}**: {optional}{fmtTyp(node)}{fmtUnits(node)}\n") - printDoc(indent + INDENTATION_UNIT, ns, node) - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) - - -def printIfDeprecated(ns, node, indent): - deprecated = node.get("deprecated", None) - if deprecated is not None: - print(f"\n{indent}.. index:: deprecated\n") - print(f"\n{indent}**DEPRECATED**: {deprecated}\n") - - -def printFullTree(ns, parent, name, indent, parent_path): - """ - recursively print the full tree structure - - :param dict ns: dictionary of namespaces for use in XPath expressions - :param lxml_element_node parent: parent node to be documented - :param str name: name of elements, such as NXentry/NXuser - :param indent: to keep track of indentation level - :param parent_path: NX class path of parent nodes - """ - - for node in parent.xpath("nx:field", namespaces=ns): - name = node.get("name") - index_name = name - dims = analyzeDimensions(ns, node) - - optional_text = get_required_or_optional_text(node) - print(f"{indent}{hyperlinkTarget(parent_path, name, 'field')}") - print(f"{indent}.. index:: {index_name} (field)\n") - print( - f"{indent}**{name}**: " - f"{optional_text}" - f"{fmtTyp(node)}" - f"{dims}" - f"{fmtUnits(node)}" - "\n" - ) - - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) - - node_list = node.xpath("nx:enumeration", namespaces=ns) - if len(node_list) == 1: - printEnumeration(indent + INDENTATION_UNIT, ns, node_list[0]) - - for subnode in node.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, - "field", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) - - for node in parent.xpath("nx:group", namespaces=ns): - name = node.get("name", "") - typ = node.get("type", "untyped (this is an error; please report)") - - optional_text = get_required_or_optional_text(node) - if typ.startswith("NX"): - if name == "": - name = typ.lstrip("NX").upper() - typ = ":ref:`%s`" % typ - hTarget = hyperlinkTarget(parent_path, name, "group") - target = hTarget.replace(".. _", "").replace(":\n", "") - # TODO: https://github.com/nexusformat/definitions/issues/1057 - print(f"{indent}{hTarget}") - print(f"{indent}**{name}**: {optional_text}{typ}\n") - - printIfDeprecated(ns, node, indent + INDENTATION_UNIT) - printDoc(indent + INDENTATION_UNIT, ns, node) - - for subnode in node.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, - "group", - subnode, - optional, - indent + INDENTATION_UNIT, - parent_path + "/" + name, - ) - - nodename = "%s/%s" % (name, node.get("type")) - printFullTree( - ns, node, nodename, indent + INDENTATION_UNIT, parent_path + "/" + name - ) - - for node in parent.xpath("nx:link", namespaces=ns): - name = node.get("name") - print(f"{indent}{hyperlinkTarget(parent_path, name, 'link')}") - print( - f"{indent}**{name}**: " - ":ref:`link` " - f"(suggested target: ``{node.get('target')}``" - "\n" - ) - printDoc(indent + INDENTATION_UNIT, ns, node) - - -def print_rst_from_nxdl(nxdl_file): - """ - print restructured text from the named .nxdl.xml file - """ - global use_application_defaults - - # parse input file into tree - tree = lxml.etree.parse(nxdl_file) - - # The following URL is outdated, but that doesn't matter; - # it won't be accessed; it's just an arbitrary namespace name. - # It only needs to match the xmlns attribute in the NXDL files. - NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" - ns = {"nx": NAMESPACE} - - root = tree.getroot() - name = root.get("name") - title = name - parent_path = "/" + name # absolute path of parent nodes, no trailing / - if len(name) < 2 or name[0:2] != "NX": - raise Exception('Unexpected class name "%s"; does not start with NX' % (name)) - lexical_name = name[2:] # without padding 'NX', for indexing - - category = root.attrib["category"] - - # Pass these terms to construct the full URL - anchor_registry.nxdl_file = nxdl_file - nxdl_subdir = os.path.basename(os.path.dirname(os.path.abspath(nxdl_file))) - anchor_registry.nxdl_subdir = nxdl_subdir - - listing_category = { - "base": "base class", - "application": "application definition", - }[category] - - use_application_defaults = category == "application" - - # print ReST comments and section header - print( - f".. auto-generated by script {sys.argv[0]} " - f"from the NXDL source {sys.argv[1]}" - ) - print("") - print(".. index::") - print(f" ! {name} ({listing_category})") - print(f" ! {lexical_name} ({listing_category})") - print(f" see: {lexical_name} ({listing_category}); {name}") - print("") - print(f".. _{name}:\n") - print("=" * len(title)) - print(title) - print("=" * len(title)) - - # print category & parent class - extends = root.get("extends") - if extends is None: - extends = "none" - else: - extends = ":ref:`%s`" % extends - - print("") - print("**Status**:\n") - print(f" {listing_category.strip()}, extends {extends}") - - printIfDeprecated(ns, root, "") - - # print official description of this class - print("") - print("**Description**:\n") - printDoc(INDENTATION_UNIT, ns, root, required=True) - - # print symbol list - node_list = root.xpath("nx:symbols", namespaces=ns) - print("**Symbols**:\n") - if len(node_list) == 0: - print(" No symbol table\n") - elif len(node_list) > 1: - raise Exception("Invalid symbol table in " % root.get("name")) - else: - printDoc(INDENTATION_UNIT, ns, node_list[0]) - for node in node_list[0].xpath("nx:symbol", namespaces=ns): - doc = getDocLine(ns, node) - print(f" **{node.get('name')}**", end="") - if doc: - print(f": {doc}", end="") - print("\n") - - # print group references - print("**Groups cited**:") - node_list = root.xpath("//nx:group", namespaces=ns) - groups = [] - for node in node_list: - g = node.get("type") - if g.startswith("NX") and g not in groups: - groups.append(g) - if len(groups) == 0: - print(" none\n") - else: - out = [(":ref:`%s`" % g) for g in groups] - txt = ", ".join(sorted(out)) - print(f" {txt}\n") - out = [("%s (base class); used in %s" % (g, listing_category)) for g in groups] - txt = ", ".join(out) - print(f".. index:: {txt}\n") - - # TODO: change instances of \t to proper indentation - - # print full tree - print("**Structure**:\n") - for subnode in root.xpath("nx:attribute", namespaces=ns): - optional = get_required_or_optional_text(subnode) - printAttribute( - ns, "file", subnode, optional, INDENTATION_UNIT, parent_path - ) # FIXME: +"/"+name ) - printFullTree(ns, root, name, INDENTATION_UNIT, parent_path) - - printAnchorList() - - # print NXDL source location - print("") - print("**NXDL Source**:") - print(f" {HTML_ROOT}/{nxdl_subdir}/{name}.nxdl.xml") - - -def main(): - """ - standard command-line processing - """ - import argparse - - parser = argparse.ArgumentParser(description="test nxdl2rst code") - parser.add_argument("nxdl_file", help="name of NXDL file") - results = parser.parse_args() - nxdl_file = results.nxdl_file - - if not os.path.exists(nxdl_file): - print(f"Cannot find {nxdl_file}") - exit() - - print_rst_from_nxdl(nxdl_file) - - # if the NXDL has a subdirectory, - # copy that subdirectory (quietly) to the pwd, such as: - # contributed/NXcanSAS.nxdl.xml: cp -a contributed/canSAS ./ - category = os.path.basename(os.getcwd()) - path = os.path.join("../../../../", category) - basename = os.path.basename(nxdl_file) - corename = basename[2:].split(".")[0] - source = os.path.join(path, corename) - if os.path.exists(source): - target = os.path.join(".", corename) - replicate(source, target) - - -if __name__ == "__main__": - WRITE_ANCHOR_REGISTRY = True - main() - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/nxdl_desc2rst.py b/utils/nxdl_desc2rst.py deleted file mode 100755 index 5d9a49a22c..0000000000 --- a/utils/nxdl_desc2rst.py +++ /dev/null @@ -1,518 +0,0 @@ -#!/usr/bin/env python - -""" -Read the NXDL field types specification and find -all the valid data types. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import os, sys -import lxml.etree -import textwrap - - -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 - - -ELEMENT_DICT = { - "attribute": """ -An ``attribute`` element can *only* be a child of a -``field`` or ``group`` element. -It is used to define *attribute* elements to be used and their data types -and possibly an enumeration of allowed values. - -For more details, see: -:ref:`NXDL.data.type.attributeType` - """, - "definition": """ -A ``definition`` element can *only* be used -at the root level of an NXDL specification. -Note: Due to the large number of attributes of the ``definition`` element, -they have been omitted from the figure below. - -For more details, see: -:ref:`NXDL.data.type.definition`, -:ref:`NXDL.data.type.definitionType`, and -:ref:`NXDL.data.type.definitionTypeAttr` - """, - "dimensions": """ -The ``dimensions`` element describes the *shape* of an array. -It is used *only* as a child of a ``field`` element. - -For more details, see: -:ref:`NXDL.data.type.dimensionsType` - """, - "doc": """ -A ``doc`` element can be a child of most NXDL elements. In most cases, the -content of the ``doc`` element will also become part of the NeXus manual. - -:element: {any}: - -In documentation, it may be useful to -use an element that is not directly specified by the NXDL language. -The *any* element here says that one can use any element -at all in a ``doc`` element and NXDL will not process it but pass it through. - -For more details, see: -:ref:`NXDL.data.type.docType` - """, - "enumeration": """ -An ``enumeration`` element can *only* be a child of a -``field`` or ``attribute`` element. -It is used to restrict the available choices to a predefined list, -such as to control varieties in spelling of a controversial word (such as -*metre* vs. *meter*). - -For more details, see: -:ref:`NXDL.data.type.enumerationType` - """, - "field": """ -The ``field`` element provides the value of a named item. Many different attributes -are available to further define the ``field``. Some of the attributes are not -allowed to be used together (such as ``axes`` and ``axis``); see the documentation -of each for details. -It is used *only* as a child of a ``group`` element. - -For more details, see: -:ref:`NXDL.data.type.fieldType` - """, - "choice": """ -A ``choice`` element is used when a named group might take one -of several possible NeXus base classes. Logically, it must -have at least two group children. - -For more details, see: -:ref:`NXDL.data.type.choiceType` - """, - "group": """ -A ``group`` element can *only* be a child of a -``definition`` or ``group`` element. -It describes a common level of organization in a NeXus data file, similar -to a subdirectory in a file directory tree. - -For more details, see: -:ref:`NXDL.data.type.groupType` - """, - "link": """ -.. index:: - single: link target - -A ``link`` element can *only* be a child of a -``definition``, -``field``, or ``group`` element. -It describes the path to the original source of the parent -``definition``, -``field``, or ``group``. - -For more details, see: -:ref:`NXDL.data.type.linkType` - """, - "symbols": """ -A ``symbols`` element can *only* be a child of a ``definition`` element. -It defines the array index symbols to be used when defining arrays as -``field`` elements with common dimensions and lengths. - -For more details, see: -:ref:`NXDL.data.type.symbolsType` - """, -} - -DATATYPE_DICT = { - "basicComponent": """/xs:schema//xs:complexType[@name='basicComponent']""", - "validItemName": """/xs:schema//xs:simpleType[@name='validItemName']""", - "validNXClassName": """/xs:schema//xs:simpleType[@name='validNXClassName']""", - "validTargetName": """/xs:schema//xs:simpleType[@name='validTargetName']""", - "nonNegativeUnbounded": """/xs:schema//xs:simpleType[@name='nonNegativeUnbounded']""", -} - -ELEMENT_PREAMBLE = """ -============================= -NXDL Elements and Field Types -============================= - -The documentation in this section has been obtained directly -from the NXDL Schema file: *nxdl.xsd*. -First, the basic elements are defined in alphabetical order. -Attributes to an element are indicated immediately following the element -and are preceded with an "@" symbol, such as -**@attribute**. -Then, the common data types used within the NXDL specification are defined. -Pay particular attention to the rules for *validItemName* -and *validNXClassName*. - -.. - 2010-11-29,PRJ: - This contains a lot of special case code to lay out the NXDL chapter. - It could be cleaner but that would also involve some cooperation on - anyone who edits nxdl.xsd which is sure to break. The special case ensures - the parts come out in the chosen order. BUT, it is possible that new - items in nxdl.xsd will not automatically go in the manual. - Can this be streamlined with some common methods? - Also, there is probably too much documentation in nxdl.xsd. Obscures the function. - -.. index:: - see: attribute; NXDL attribute - ! single: NXDL elements - -.. _NXDL.elements: - -NXDL Elements -============= - - """ - -DATATYPE_PREAMBLE = """ - -.. _NXDL.data.types.internal: - -NXDL Field Types (internal) -=========================== - -Field types that define the NXDL language are described here. -These data types are defined in the XSD Schema (``nxdl.xsd``) -and are used in various parts of the Schema to define common structures -or to simplify a complicated entry. While the data types are not intended for -use in NXDL specifications, they define structures that may be used in NXDL specifications. - -""" - -DATATYPE_POSTAMBLE = """ -**The** ``xs:string`` **data type** - The ``xs:string`` data type can contain characters, - line feeds, carriage returns, and tab characters. - See https://www.w3schools.com/xml/schema_dtypes_string.asp - for more details. - -**The** ``xs:token`` **data type** - The ``xs:string`` data type is derived from the - ``xs:string`` data type. - - The ``xs:token`` data type also contains characters, - but the XML processor will remove line feeds, carriage returns, tabs, - leading and trailing spaces, and multiple spaces. - See https://www.w3schools.com/xml/schema_dtypes_string.asp - for more details. -""" - - -def _tagMatch(ns, parent, match_list): - """match this tag to a list""" - if parent is None: - raise ValueError("Must supply a valid parent node") - parent_tag = parent.tag - tag_found = False - for item in match_list: - # this routine only handles certain XML Schema components - tag_found = parent_tag == "{%s}%s" % (ns["xs"], item) - if tag_found: - break - return tag_found - - -def _indent(indentLevel): - return INDENTATION * indentLevel - - -def printTitle(title, indentLevel): - print(title) - print(TITLE_MARKERS[indentLevel] * len(title) + "\n") - - -def generalHandler(ns, parent=None, indentLevel=0): - """Handle XML nodes like the former XSLT template""" - # ignore things we don't know how to handle - known_tags = ("complexType", "simpleType", "group", "element", "attribute") - if not _tagMatch(ns, parent, known_tags): - return - - parent_name = parent.get("name") - if parent_name is None: - return - - simple_tag = parent.tag[ - parent.tag.find("}") + 1 : - ] # cut off the namespace identifier - - # ... - name = parent_name # + ' data type' - if simple_tag == "attribute": - name = "@" + name - - if indentLevel == 0 and not simple_tag in ("attribute"): - print(".. index:: ! %s (NXDL data type)\n" % name) - print("\n.. _%s:\n" % ("NXDL.data.type." + name)) - - printTitle(name, indentLevel) - - printDocs(ns, parent, indentLevel) - - if len(parent.xpath("xs:attribute", namespaces=ns)) > 0: - printTitle("Attributes of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:attribute", indentLevel + 1) - - node_list = parent.xpath("xs:restriction", namespaces=ns) - if len(node_list) > 0: - # printTitle("Restrictions of "+name, indentLevel+1) - restrictionHandler(ns, node_list[0], indentLevel + 1) - node_list = parent.xpath( - "xs:simpleType/xs:restriction/xs:enumeration", namespaces=ns - ) - if len(node_list) > 0: - # printTitle("Enumerations of "+name, indentLevel+1) - applyTemplates( - ns, - parent, - "xs:simpleType/xs:restriction", - indentLevel + 1, - handler=restrictionHandler, - ) - - if len(parent.xpath("xs:sequence/xs:element", namespaces=ns)) > 0: - printTitle("Elements of " + name, indentLevel + 1) - applyTemplates(ns, parent, "xs:sequence/xs:element", indentLevel + 1) - - node_list = parent.xpath("xs:sequence/xs:group", namespaces=ns) - if len(node_list) > 0: - printTitle("Groups under " + name, indentLevel + 1) - printDocs(ns, node_list[0], indentLevel + 1) - - applyTemplates(ns, parent, "xs:simpleType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType", indentLevel + 1) - applyTemplates(ns, parent, "xs:complexType/xs:attribute", indentLevel + 1) - applyTemplates( - ns, parent, "xs:complexContent/xs:extension/xs:attribute", indentLevel + 1 - ) - applyTemplates( - ns, parent, "xs:complexType/xs:sequence/xs:attribute", indentLevel + 1 - ) - applyTemplates(ns, parent, "xs:complexType/xs:sequence/xs:element", indentLevel + 1) - applyTemplates( - ns, - parent, - "xs:complexContent/xs:extension/xs:sequence/xs:element", - indentLevel + 1, - ) - - -def restrictionHandler(ns, parent=None, indentLevel=0): - """Handle XSD restriction nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ("restriction",)): - return - printDocs(ns, parent, indentLevel) - print("\n") - print(_indent(indentLevel) + "The value may be any") - base = parent.get("base") - pattern_nodes = parent.xpath("xs:pattern", namespaces=ns) - enumeration_nodes = parent.xpath("xs:enumeration", namespaces=ns) - if len(pattern_nodes) > 0: - print( - _indent(indentLevel) - + "``%s``" % base - + " that *also* matches the regular expression::\n" - ) - print(_indent(indentLevel) + " " * 4 + pattern_nodes[0].get("value")) - elif len(pattern_nodes) > 0: - # how will this be reached? Perhaps a deprecated procedure - print(_indent(indentLevel) + "``%s``" % base + " from this list:") - for node in enumeration_nodes: - enumerationHandler(ns, node, indentLevel) - printDocs(ns, node, indentLevel) - print(_indent(indentLevel)) - elif len(enumeration_nodes) > 0: - print(_indent(indentLevel) + "one from this list only:\n") - for node in enumeration_nodes: - enumerationHandler(ns, node, indentLevel) - printDocs(ns, parent, indentLevel) - print(_indent(indentLevel)) - else: - print("@" + base) - print("\n") - - -def enumerationHandler(ns, parent=None, indentLevel=0): - """Handle XSD enumeration nodes like the former XSLT template""" - if not _tagMatch(ns, parent, ["enumeration"]): - return - print(_indent(indentLevel) + "* ``%s``" % parent.get("value")) - printDocs(ns, parent, indentLevel) - - -def applyTemplates(ns, parent, path, indentLevel, handler=generalHandler): - """iterate the nodes found on the supplied XPath expression""" - db = {} - for node in parent.xpath(path, namespaces=ns): - name = node.get("name") or node.get("ref") or node.get("value") - if name is not None: - if name in ("nx:groupGroup",): - print(">" * 45, name) - if name in db: - raise KeyError("Duplicate name found: " + name) - db[name] = node - for name in sorted(db): - node = db[name] - handler(ns, node, indentLevel) - # printDocs(ns, node, indentLevel) - - -def printDocs(ns, parent, indentLevel=0): - docs = getDocFromNode(ns, parent) - if docs is not None: - print(_indent(indentLevel) + "\n") - for line in docs.splitlines(): - print(_indent(indentLevel) + line) - print(_indent(indentLevel) + "\n") - - -def getDocFromNode(ns, node, retval=None): - annotation_node = node.find("xs:annotation", ns) - if annotation_node is None: - return retval - documentation_node = annotation_node.find("xs:documentation", ns) - if documentation_node is None: - return retval - - # Be sure to grab _all_ content in the node. - # In the documentation nodes, use XML entities ("<"" instead of "<") - # for documentation characters that would otherwise be considered as XML. - s = lxml.etree.tostring(documentation_node, method="text", pretty_print=True) - rst = s.decode().lstrip("\n") # remove any leading blank lines - rst = rst.rstrip() # remove any trailing white space - text = textwrap.dedent(rst) # remove common leading space - - # substitute HTML entities in markup: "<" for "<" - # thanks: http://stackoverflow.com/questions/2087370/decode-html-entities-in-python-string - try: # see #661 - import html - - text = html.unescape(text) - except (ImportError, AttributeError): - from html import parser as HTMLParser - - htmlparser = HTMLParser.HTMLParser() - text = htmlparser.unescape(text) - - return text.lstrip() - - -def addFigure(name, indentLevel=0): - fmt = """ -.. compound:: - - .. _%s: - - .. figure:: %s - :alt: fig.nxdl/nxdl_%s - :width: %s - - Graphical representation of the NXDL ``%s`` element - - .. Images of NXDL structure are generated from nxdl.xsd source - using the Eclipse XML Schema Editor (Web Tools Platform). Open the nxdl.xsd file and choose the - "Design" tab. Identify the structure to be documented and double-click to expand - as needed to show the detail. Use the XSD > "Export Diagram as Image ..." menu item (also available - as button in top toolbar). - Set the name: "nxdl_%s.png" and move the file into the correct location using - your operating system's commands. Commit the revision to version control. - """ - imageFile = "img/nxdl/nxdl_%s.png" % name - figure_id = "fig.nxdl_%s" % name - if not os.path.exists(os.path.abspath(imageFile)): - return - text = fmt % (figure_id, imageFile, name, "80%", name, name,) - indent = _indent(indentLevel) - for line in text.splitlines(): - print(indent + line) - print("\n") - - -def pickNodesFromXpath(ns, parent, path): - return parent.xpath(path, namespaces=ns) - - -def main(tree, ns): - print(".. auto-generated by script: " + sys.argv[0]) - print(ELEMENT_PREAMBLE) - - for name in sorted(ELEMENT_DICT): - print("") - print(".. index:: ! %s (NXDL element)\n" % name) - print(".. _%s:\n" % name) - printTitle(name, indentLevel=0) - print("\n") - print(ELEMENT_DICT[name]) - print("\n") - addFigure(name, indentLevel=0) - - print(DATATYPE_PREAMBLE) - - path_list = ( - "/xs:schema/xs:complexType[@name='attributeType']", - "/xs:schema/xs:element[@name='definition']", - "/xs:schema/xs:complexType[@name='definitionType']", - "/xs:schema/xs:simpleType[@name='definitionTypeAttr']", - "/xs:schema/xs:complexType[@name='dimensionsType']", - "/xs:schema/xs:complexType[@name='docType']", - "/xs:schema/xs:complexType[@name='enumerationType']", - "/xs:schema/xs:complexType[@name='fieldType']", - "/xs:schema/xs:complexType[@name='choiceType']", - "/xs:schema/xs:complexType[@name='groupType']", - "/xs:schema/xs:complexType[@name='linkType']", - "/xs:schema/xs:complexType[@name='symbolsType']", - "/xs:schema/xs:complexType[@name='basicComponent']", - "/xs:schema/xs:simpleType[@name='validItemName']", - "/xs:schema/xs:simpleType[@name='validNXClassName']", - "/xs:schema/xs:simpleType[@name='validTargetName']", - "/xs:schema/xs:simpleType[@name='nonNegativeUnbounded']", - ) - for path in path_list: - nodes = pickNodesFromXpath(ns, tree, path) - print("\n.. Xpath = %s\n" % path) - generalHandler(ns, parent=nodes[0]) - - print(DATATYPE_POSTAMBLE) - - -if __name__ == "__main__": - developermode = True - developermode = False - if developermode and len(sys.argv) != 2: - path = os.path.dirname(__file__) - NXDL_SCHEMA_FILE = os.path.join(path, "..", "nxdl.xsd") - else: - if len(sys.argv) != 2: - print("usage: %s nxdl.xsd" % sys.argv[0]) - exit() - NXDL_SCHEMA_FILE = sys.argv[1] - if not os.path.exists(NXDL_SCHEMA_FILE): - print("Cannot find %s" % NXDL_SCHEMA_FILE) - exit() - - tree = lxml.etree.parse(NXDL_SCHEMA_FILE) - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - - main(tree, ns) - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/nxdl_summary.py b/utils/nxdl_summary.py deleted file mode 100755 index a8cbc67162..0000000000 --- a/utils/nxdl_summary.py +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env python - -""" -Summarize the NXDL classes definitions for the given NXDL section. - -Re-write the index.rst file with a list of: class summary (and a hidden toctree) -""" - - -import os, sys -import lxml.etree - - -TITLE_MARKERS = "- + ~ ^ * @".split() # used for underscoring section titles -INDENTATION = " " * 4 - - -NAMESPACE = "http://definition.nexusformat.org/nxdl/3.1" -NS = {"nx": NAMESPACE} - - -PREAMBLES = { - "base_classes": """ -.. index:: - ! see: class definitions; base class - ! base class - -.. _base.class.definitions: - -Base Class Definitions -###################### - -A description of each NeXus base class definition is given. -NeXus base class definitions define the set of terms that -*might* be used in an instance of that class. -Consider the base classes as a set of *components* -that are used to construct a data file. - """, - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "applications": """ -.. index:: - ! see: class definitions; application definition - ! application definition - -.. _application.definitions: - -Application Definitions -######################### - -A description of each NeXus application definition is given. -NeXus application definitions define the *minimum* -set of terms that -*must* be used in an instance of that class. -Application definitions also may define terms that -are optional in the NeXus data file. The definition, in this case, -reserves the exact term by declaring its spelling and description. -Consider an application definition as a *contract* -between a data provider (such as the beam line control system) and a -data consumer (such as a data analysis program for a scientific technique) -that describes the information is certain to be available in a data file. - -Use NeXus links liberally in data files to reduce duplication of data. -In application definitions involving raw data, -write the raw data in the :ref:`NXinstrument` tree and then link to it -from the location(s) defined in the relevant application definition. - """, - # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - "contributed_definitions": """ -.. index:: - ! see: class definitions; contributed definition - ! contributed definition - -.. _contributed.definitions: - -Contributed Definitions -######################### - -A description of each NeXus contributed definition is given. -NXDL files in the NeXus contributed definitions include propositions from -the community for NeXus base classes or application definitions, as well -as other NXDL files for long-term archival by NeXus. Consider the contributed -definitions as either in *incubation* or a special -case not for general use. The :ref:`NIAC` is charged to review any new contributed -definitions and provide feedback to the authors before ratification -and acceptance as either a base class or application definition. - """, -} - - -def getSummary(nxdl_file): - """ - get the summary line from each NXDL definition doc - - That's the first physical line of the doc string. - """ - tree = lxml.etree.parse(nxdl_file) - root = tree.getroot() - nodes = root.xpath("nx:doc", namespaces=NS) - if len(nodes) != 1: - raise RuntimeError("wrong number of nodes in NXDL: " + nxdl_file) - text = nodes[0].text - return text.strip().splitlines()[0] - - -def command_args(): - """get the command-line arguments, handle syntax errors""" - import argparse - - doc = __doc__.strip().splitlines()[0] - parser = argparse.ArgumentParser(prog=sys.argv[0], description=doc) - parser.add_argument( - "section", action="store", help="NXDL section (such as *base_classes*)" - ) - return parser.parse_args() - - -def main(section): - if section not in PREAMBLES.keys(): - raise KeyError("unknown NXDL section: " + section) - base_path = os.path.abspath(os.path.dirname(__file__)) - nxdl_path = os.path.abspath(os.path.join(base_path, "..", section)) - if not os.path.exists(nxdl_path): - raise IOError("not found: " + nxdl_path) - - rst_path = os.path.abspath( - os.path.join(base_path, "..", "manual", "source", "classes", section) - ) - if not os.path.exists(rst_path): - raise IOError("not found: " + rst_path) - - index_file = os.path.join(rst_path, "index.rst") - - classes = [] - text = [] - text.append( - """ -.. do NOT edit this file - automatically generated by script """ - + __file__ - ) - text.append("") - text.append(PREAMBLES[section]) - for fname in sorted(os.listdir(nxdl_path)): - if fname.endswith(".nxdl.xml"): - class_name = fname.split(".")[0] - classes.append(class_name) - summary = getSummary(os.path.join(nxdl_path, fname)) - text.append("") - text.append(":ref:`" + class_name + "`") - text.append(INDENTATION + summary) - text.append("") - text.append(".. toctree::") - text.append(INDENTATION + ":hidden:") - text.append("") - for cname in sorted(classes): - text.append(INDENTATION + cname) - text.append("") - open(index_file, "w").writelines("\n".join(text)) - - -if __name__ == "__main__": - cli = command_args() - main(cli.section) diff --git a/utils/test_nxdl.py b/utils/test_nxdl.py deleted file mode 100644 index e7152c7941..0000000000 --- a/utils/test_nxdl.py +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/env python - -""" -unit testing of NeXus definitions NXDL files and XML Schema -""" - -import os -import sys -import unittest -import lxml.etree - -# xmllint --noout --schema nxdl.xsd base_classes/NXentry.nxdl.xml -# base_classes/NXentry.nxdl.xml validates - - -BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) -NXDL_XSD_SCHEMA = "nxdl.xsd" -NXDL_SCHEMA = lxml.etree.XMLSchema( - lxml.etree.parse(os.path.join(BASE_DIR, NXDL_XSD_SCHEMA)) -) - -NXDL_CATEGORY_NAMES = "base_classes applications contributed_definitions".split() - - -class NXDL_Invalid(Exception): - pass - - -class NXDL_Valid(Exception): - pass - - -def isNXDL(fname): - return fname.endswith(".nxdl.xml") - - -def get_NXDL_file_list(): - os.chdir(BASE_DIR) - file_list = [] - for category in NXDL_CATEGORY_NAMES: - raw_list = os.listdir(category) - nxdl_files = [os.path.join(category, fn) for fn in raw_list if isNXDL(fn)] - file_list += sorted(nxdl_files) - return file_list - - -def validate_xml(xml_file_name): - """ - validate an NXDL XML file against an XML Schema file - - :param str xml_file_name: name of XML file - """ - try: - xml_tree = lxml.etree.parse(xml_file_name) - except lxml.etree.XMLSyntaxError as exc: - msg = xml_file_name + " : " + str(exc) - raise NXDL_Invalid(msg) - try: - result = NXDL_SCHEMA.assertValid(xml_tree) - # there is no assertNotRaises so raise this when successful - raise NXDL_Valid - except lxml.etree.DocumentInvalid as exc: - msg = xml_file_name + " : " + str(exc) - raise NXDL_Invalid(msg) - - -class TestMaker(type): - def __new__(cls, clsname, bases, dct): - # Add a method to the class' __dict__ for every - # file name in the NXDL file list. - cat_number_dict = {c: str(i + 1) for i, c in enumerate(NXDL_CATEGORY_NAMES)} - for fname in get_NXDL_file_list(): - category, nxdl_name = os.path.split(fname) - category_number = cat_number_dict[category] - point = nxdl_name.find(".") - nxdl_name = nxdl_name[:point] - test_name = "test" - # since these will be sorted, get the categories in the desired order - test_name += "__" + str(category_number) - test_name += "__" + category - test_name += "__" + nxdl_name - dct[test_name] = cls.make_test(fname) - - return super(TestMaker, cls).__new__(cls, clsname, bases, dct) - - @staticmethod - def make_test(nxdl_file_name): - def test_wrap(self): - # test body for each NXDL file test - with self.assertRaises(NXDL_Valid): - validate_xml(nxdl_file_name) - self.assertRaises(NXDL_Valid, validate_xml, nxdl_file_name) - - return test_wrap - - -class Individual_NXDL_Tests(unittest.TestCase, metaclass=TestMaker): - """ - run all tests created in TestMaker() class, called by suite() - """ - - -def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" - test_suite = unittest.TestSuite() - test_suite.addTests(unittest.makeSuite(Individual_NXDL_Tests)) - return test_suite - - -def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) - runner.run(suite()) - - -if __name__ == "__main__": - run() diff --git a/utils/test_nxdl2rst.py b/utils/test_nxdl2rst.py deleted file mode 100644 index 805c02ae87..0000000000 --- a/utils/test_nxdl2rst.py +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/env python - -""" -unit testing: NXDL to RST for documentation -""" - -import os -import sys -import unittest -import lxml.etree -from io import StringIO - -import nxdl2rst - - -class Capture_stdout(list): - """ - capture all printed output (to stdout) into list - - # http://stackoverflow.com/questions/16571150/how-to-capture-stdout-output-from-a-python-function-call - """ - - def __enter__(self): - self._stdout = sys.stdout - sys.stdout = self._stringio = StringIO() - return self - - def __exit__(self, *args): - self.extend(self._stringio.getvalue().splitlines()) - del self._stringio # free up some memory - sys.stdout = self._stdout - - -class Issue_524_Clarify_Optional_or_Required(unittest.TestCase): - """ - make it obvious what is required and what is optional - - **field**: (optional or required) NX_TYPE - """ - - def test_base_class_NXentry(self): - expected_lines = """ - **definition**: (optional) :ref:`NX_CHAR ` - **DATA**: (optional) :ref:`NXdata` - **notes**: (optional) :ref:`NXnote` - **@default**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXentry", expected_lines) - - def test_base_class_NXuser(self): - expected_lines = """ - **name**: (optional) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("base_classes", "NXuser", expected_lines) - - def test_application_definition_NXcanSAS(self): - expected_lines = """ - **definition**: (required) :ref:`NX_CHAR ` - **title**: (required) :ref:`NX_CHAR ` - **run**: (required) :ref:`NX_CHAR ` - **I**: (required) :ref:`NX_NUMBER ` - **Q**: (required) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **Idev**: (optional) :ref:`NX_NUMBER ` - **dQw**: (optional) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **dQl**: (optional) :ref:`NX_NUMBER ` {units=\ :ref:`NX_PER_LENGTH `} - **ENTRY**: (required) :ref:`NXentry` - **DATA**: (required) :ref:`NXdata` - **TRANSMISSION_SPECTRUM**: (optional) :ref:`NXdata` - **SAMPLE**: (optional) :ref:`NXsample` - **INSTRUMENT**: (optional) :ref:`NXinstrument` - **NOTE**: (optional) :ref:`NXnote` - **PROCESS**: (optional) :ref:`NXprocess` - **SOURCE**: (optional) :ref:`NXsource` - **@default**: (optional) :ref:`NX_CHAR ` - **@timestamp**: (optional) :ref:`NX_DATE_TIME ` - **@canSAS_class**: (required) :ref:`NX_CHAR ` - **@signal**: (required) :ref:`NX_CHAR ` - **@I_axes**: (required) :ref:`NX_CHAR ` - """.strip().splitlines() - - self.apply_tests("applications", "NXcanSAS", expected_lines) - - def apply_tests(self, category, class_name, expected_lines): - nxdl_file = os.path.join( - os.path.dirname(__file__), "..", category, class_name + ".nxdl.xml" - ) - self.assertTrue(os.path.exists(nxdl_file), nxdl_file) - - sys.argv.insert(0, "python") - with Capture_stdout() as printed_lines: - nxdl2rst.print_rst_from_nxdl(nxdl_file) - - printed_lines = [_.strip() for _ in printed_lines] - for line in expected_lines: - expected = line.strip() - self.assertTrue(expected in printed_lines, line.strip()) - - -def suite(*args, **kw): - """gather all the tests together in a suite, called by run()""" - test_suite = unittest.TestSuite() - test_suite_list = [ - Issue_524_Clarify_Optional_or_Required, - ] - for item in test_suite_list: - test_suite.addTests(unittest.makeSuite(item)) - return test_suite - - -def run(): - """run all the unit tests""" - runner = unittest.TextTestRunner(verbosity=2) - runner.run(suite()) - - -if __name__ == "__main__": - run() diff --git a/utils/test_suite.py b/utils/test_suite.py deleted file mode 100644 index 92b8bb246b..0000000000 --- a/utils/test_suite.py +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env python - -""" -unit testing of the NeXus definitions -""" - -import os -import unittest -import sys - - -def suite(*args, **kw): - import test_nxdl - import test_nxdl2rst - - test_suite = unittest.TestSuite() - test_list = [ - test_nxdl, - test_nxdl2rst, - ] - - for test in test_list: - test_suite.addTest(test.suite()) - return test_suite - - -if __name__ == "__main__": - owd = os.getcwd() - runner = unittest.TextTestRunner(verbosity=2) - result = runner.run(suite()) - os.chdir(owd) - sys.exit(len(result.errors)) diff --git a/utils/types2rst.py b/utils/types2rst.py deleted file mode 100755 index 2faa1a62fc..0000000000 --- a/utils/types2rst.py +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python - -""" -Read the the NeXus NXDL types specification and find -all the valid data types. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import units2rst - - -if __name__ == "__main__": - units2rst.worker("primitiveType", section="data") - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/units2rst.py b/utils/units2rst.py deleted file mode 100755 index ad10ddfeb4..0000000000 --- a/utils/units2rst.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/env python - -""" -Read the the NeXus NXDL types specification and find -all the valid types of units. Write a restructured -text (.rst) document for use in the NeXus manual in -the NXDL chapter. -""" - - -import os, sys -import lxml.etree - - -def worker(nodeMatchString, section="units"): - if len(sys.argv) != 2: - print("usage: %s nxdlTypes.xsd" % sys.argv[0]) - exit() - NXDL_TYPES_FILE = sys.argv[1] - if not os.path.exists(NXDL_TYPES_FILE): - print("Cannot find %s" % NXDL_TYPES_FILE) - exit() - - tree = lxml.etree.parse(NXDL_TYPES_FILE) - - output = [".. auto-generated by %s -- DO NOT EDIT" % sys.argv[0]] - output.append("") - - labels = ("term", "description") - output.append(".. nodeMatchString : %s" % nodeMatchString) - output.append("") - db = {} - - NAMESPACE = "http://www.w3.org/2001/XMLSchema" - ns = {"xs": NAMESPACE} - root = tree.xpath("//xs:schema", namespaces=ns)[0] - s = "//xs:simpleType" - node_list = tree.xpath("//xs:simpleType", namespaces=ns) - - # get the names of all the types of units - members = [] - for node in node_list: - if node.get("name") == nodeMatchString: - union = node.xpath("xs:union", namespaces=ns) - members = union[0].get("memberTypes", "").split() - - # get the definition of each type of units - for node in node_list: - node_name = node.get("name") - if node_name is None: - continue - if "nxdl:" + node_name in members: - words = node.xpath("xs:annotation/xs:documentation", namespaces=ns)[0] - examples = [] - for example in words.iterchildren(): - nm = example.attrib.get("name") - if nm is not None and nm == "example": - examples.append("``" + example.text + "``") - a = words.text - if len(examples) > 0: - a = " ".join(a.split()) + ",\n\texample(s): " + " | ".join(examples) - db[node_name] = a - - # for item in node.xpath('xs:restriction//xs:enumeration', namespaces=ns): - # key = '%s' % item.get('value') - # words = item.xpath('xs:annotation/xs:documentation', namespaces=ns)[0] - # db[key] = words.text - - print("\n".join(output)) - - # this list is too long to make this a table in latex - # for two columns, a Sphinx fieldlist will do just as well - for key in sorted(db): - print(".. index:: ! %s (%s type)\n" % (key, section)) # index entry - print(".. _%s:\n" % key) # cross-reference point - print(":%s:" % key) - for line in db[key].splitlines(): - print(" %s" % line) - print("") - - -if __name__ == "__main__": - # sys.argv.append('../nxdlTypes.xsd') # FIXME: developer only -- remove for production!!! - worker("anyUnitsAttr") - - -# NeXus - Neutron and X-ray Common Data Format -# -# Copyright (C) 2008-2022 NeXus International Advisory Committee (NIAC) -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -# -# For further information, see http://www.nexusformat.org diff --git a/utils/update_copyright_date.py b/utils/update_copyright_date.py index af7b73fca1..261b1933a3 100755 --- a/utils/update_copyright_date.py +++ b/utils/update_copyright_date.py @@ -12,7 +12,6 @@ import os, sys import mimetypes -import local_utilities from build_preparation import ROOT_DIR_EXPECTED_RESOURCES import datetime From 15dc919da2e53d379a9574c6d8db9b75514adf33 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 29 Jun 2022 22:00:18 +0200 Subject: [PATCH 3/6] fixup README's and comments --- .github/workflows/ci.yaml | 4 ++-- BUILDING.rst | 4 ---- README.md | 8 ++++---- dev_tools/README.md | 2 +- setup.cfg | 5 ++++- 5 files changed, 11 insertions(+), 12 deletions(-) delete mode 100644 BUILDING.rst diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2187895160..05ce33b182 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -80,14 +80,14 @@ jobs: ls -lAFgh build/manual/build/html/index.html ls -lAFgh build/manual/build/latex/nexus.pdf - - name: Build and Commit the NeXus User Manual + - name: Build and Commit the User Manual if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} uses: sphinx-notes/pages@master with: # path to the conf.py directory documentation_path: build/manual/source - - name: Deploy the NeXus User Manual + - name: Deploy the User Manual if: ${{ startsWith(github.ref, 'refs/tags') && env.python_version == '3.7' }} uses: ad-m/github-push-action@master with: diff --git a/BUILDING.rst b/BUILDING.rst deleted file mode 100644 index 6912ca1e6d..0000000000 --- a/BUILDING.rst +++ /dev/null @@ -1,4 +0,0 @@ -The documentation relies on Sphinx (a Python package) for its organization. The -GNU ``make`` program and Python are used to build the NeXus documentation. The -default build assembles the HTML version. - diff --git a/README.md b/README.md index a9e2e3ac42..6bb56e3ea5 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,14 @@ * Documentation: https://manual.nexusformat.org/ * Release Notes: https://github.com/nexusformat/definitions/wiki/Release-Notes * License: [![License](https://img.shields.io/badge/License-LGPL_v3-blue.svg)](https://www.gnu.org/licenses/lgpl-3.0) -* Test, Build and Deploy: [![Workflow](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml) +* Test, Build and Deploy: [![CI](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml/badge.svg)](https://github.com/nexusformat/definitions/actions/workflows/ci.yaml) ## NeXus definition developers -After make a change to the NeXus class definitions there are two important steps -to take before commiting the change: +After making a change to the NeXus class definitions there are two important checks +to be made before commiting the change: - 1. check whether the change do not violate any syntax rules + 1. check whether the change does not violate any syntax rules 2. verify whether the change looks as intended in the HTML documentation First install the test and build requirements with this command (only run once) diff --git a/dev_tools/README.md b/dev_tools/README.md index efdc54bd3d..2ebdaacdf0 100644 --- a/dev_tools/README.md +++ b/dev_tools/README.md @@ -8,7 +8,7 @@ and generate all files needed for documentation building with sphinx. Install all requirements ```bash -python3 -m pip install -r dev_tools/requirements.txt +python3 -m pip install -r requirements.txt ``` Run all tests diff --git a/setup.cfg b/setup.cfg index 4582c6c60b..a49eb0be12 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,8 +1,11 @@ +# https://black.readthedocs.io/en/stable/guides/using_black_with_other_tools.html + # E501 (line too long) ignored for now -# E203 and W503 incompatible with black formatting (https://black.readthedocs.io/en/stable/compatible_configs.html#flake8) +# E203 and W503 incompatible with black formatting [flake8] ignore = E501, E203, W503 max-line-length = 88 [isort] +profile = black force_single_line = true \ No newline at end of file From 2372f9287500e34683a15a9d17bb31837c24a9bb Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Wed, 29 Jun 2022 22:03:06 +0200 Subject: [PATCH 4/6] fix bug in NXDL repo URL in the HTML docs and NXDL HTML URL in the anchor list --- dev_tools/apps/manual_app.py | 6 +++--- dev_tools/apps/nxclass_app.py | 10 ++-------- dev_tools/docs/anchor_list.py | 17 +++++++++++++---- dev_tools/docs/nxdl.py | 17 ++++------------- dev_tools/docs/xsd.py | 2 +- dev_tools/globals/directories.py | 14 ++++++++++---- 6 files changed, 33 insertions(+), 33 deletions(-) diff --git a/dev_tools/apps/manual_app.py b/dev_tools/apps/manual_app.py index a03dbf7bb3..768332b58d 100644 --- a/dev_tools/apps/manual_app.py +++ b/dev_tools/apps/manual_app.py @@ -64,7 +64,7 @@ def manual_exec(args): if generate_docs: xsd_file = directories.get_xsd_file() rst_lines = xsdgenerator(xsd_file) - nxdl_desc = directories.manual_build_sphinxsroot() / "nxdl_desc.rst" + nxdl_desc = directories.manual_build_sphinxroot() / "nxdl_desc.rst" if args.diff: diff_ascii(xsd_file, rst_lines, nxdl_desc) if args.prepare: @@ -75,7 +75,7 @@ def manual_exec(args): if generate_docs: xsd_file = directories.get_xsd_units_file() rst_lines = generate_xsd_units_doc(xsd_file, "anyUnitsAttr", "units") - nxdl_desc = directories.manual_build_sphinxsroot() / "units.table" + nxdl_desc = directories.manual_build_sphinxroot() / "units.table" if args.diff: diff_ascii(xsd_file, rst_lines, nxdl_desc) if args.prepare: @@ -86,7 +86,7 @@ def manual_exec(args): if generate_docs: xsd_file = directories.get_xsd_units_file() rst_lines = generate_xsd_units_doc(xsd_file, "primitiveType", "data") - nxdl_desc = directories.manual_build_sphinxsroot() / "types.table" + nxdl_desc = directories.manual_build_sphinxroot() / "types.table" if args.diff: diff_ascii(xsd_file, rst_lines, nxdl_desc) if args.prepare: diff --git a/dev_tools/apps/nxclass_app.py b/dev_tools/apps/nxclass_app.py index 9023108d69..e0025cfd59 100644 --- a/dev_tools/apps/nxclass_app.py +++ b/dev_tools/apps/nxclass_app.py @@ -70,7 +70,7 @@ def save_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: """Build the NXDL file: this means prepare the documentation and resources in the build directory. """ - rst_file_name = get_rst_filename(nxdl_file) + rst_file_name = directories.get_rst_filename(nxdl_file) # Save the documentation in the build directory print("generate", nxdl_file) @@ -99,11 +99,5 @@ def diff_nxclass_docs(nxdl_file: Path, rst_lines: List[str]) -> None: """Build the NXDL file: this means prepare the documentation and resources in the build directory. """ - rst_file_name = get_rst_filename(nxdl_file) + rst_file_name = directories.get_rst_filename(nxdl_file) diff_ascii(nxdl_file, rst_lines, rst_file_name) - - -def get_rst_filename(nxdl_file: Path) -> Path: - rst_file_name = directories.nxclass_build_root(nxdl_file) - rst_file_name /= nxdl_file.with_suffix("").with_suffix(".rst").name - return rst_file_name diff --git a/dev_tools/docs/anchor_list.py b/dev_tools/docs/anchor_list.py index 874958cec1..070927dae9 100644 --- a/dev_tools/docs/anchor_list.py +++ b/dev_tools/docs/anchor_list.py @@ -1,11 +1,13 @@ import datetime import json +import os from pathlib import Path from typing import Optional import lxml import yaml +from ..globals import directories from ..globals.nxdl import get_nxdl_version from ..globals.urls import MANUAL_URL from ..utils.types import PathLike @@ -53,7 +55,6 @@ def __init__(self, output_path: Optional[PathLike] = None) -> None: self._registry = self._load_registry() self._anchor_buffer = [] self._nxdl_file = None - self.category = None @property def all_anchors(self): @@ -70,6 +71,15 @@ def nxdl_file(self) -> Optional[Path]: def nxdl_file(self, value: PathLike) -> None: self._nxdl_file = Path(value).absolute() + @property + def html_url(self): + rst_file = directories.get_rst_filename(self.nxdl_file) + manual_root = directories.manual_build_sphinxroot() + rel_path = rst_file.relative_to(manual_root) + rel_html = str(rel_path.with_suffix(".html")) + rel_html = rel_html.replace(os.sep, "/") + return f"{MANUAL_URL}/{rel_html}" + def add(self, anchor): """Add anchor to the in-memory registry and to the current anchor buffer.""" @@ -84,8 +94,7 @@ def add(self, anchor): reg = self._registry[key] if anchor not in reg: hanchor = self._html_anchor(anchor) - fnxdl = "/".join(self.nxdl_file.parts[-2:]).split(".")[0] - url = f"{MANUAL_URL}classes/{self.category}/{fnxdl}.html{hanchor}" + url = f"{self.html_url}{hanchor}" reg[anchor] = dict( term=anchor, html=hanchor, @@ -170,7 +179,7 @@ def _write_html(self, contents): p.text = "This content is also available in these formats: " for ext in "json txt yml".split(): a = lxml.etree.SubElement(p, "a") - a.attrib["href"] = f"{MANUAL_URL}_static/{self._txt_file.stem}.{ext}" + a.attrib["href"] = f"{MANUAL_URL}/_static/{self._txt_file.stem}.{ext}" a.text = f" {ext}" dl = lxml.etree.SubElement(body, "dl") diff --git a/dev_tools/docs/nxdl.py b/dev_tools/docs/nxdl.py index 0c803482b0..a7a355cc2d 100644 --- a/dev_tools/docs/nxdl.py +++ b/dev_tools/docs/nxdl.py @@ -22,11 +22,6 @@ class NXClassDocGenerator: _INDENTATION_UNIT = " " * 2 - _CATEGORY_TO_DIRNAME = { - "base": "base_classes", - "application": "applications", - } - _CATEGORY_TO_LISTING = { "base": "base class", "application": "application definition", @@ -77,11 +72,6 @@ def _parse_nxdl_file(self, nxdl_file: Path): f'Unexpected class name "{nxclass_name}"; does not start with NX' ) lexical_name = nxclass_name[2:] # without padding 'NX', for indexing - - # Pass these terms to construct the full URL - if self._anchor_registry: - self._anchor_registry.category = self._CATEGORY_TO_DIRNAME[category] - self._listing_category = self._CATEGORY_TO_LISTING[category] self._use_application_defaults = category == "application" @@ -174,9 +164,10 @@ def _parse_nxdl_file(self, nxdl_file: Path): # print NXDL source location self._print("") self._print("**NXDL Source**:") - self._print( - f" {REPO_URL}/{self._CATEGORY_TO_DIRNAME[category]}/{nxclass_name}.nxdl.xml" - ) + nxdl_root = get_nxdl_root() + rel_path = str(nxdl_file.relative_to(nxdl_root)) + rel_html = str(rel_path).replace(os.sep, "/") + self._print(f" {REPO_URL}/{rel_html}") return self._rst_lines diff --git a/dev_tools/docs/xsd.py b/dev_tools/docs/xsd.py index c0936416d1..88c2e0fe98 100644 --- a/dev_tools/docs/xsd.py +++ b/dev_tools/docs/xsd.py @@ -284,7 +284,7 @@ def get_doc_from_node(self, node, retval=None): def add_figure(self, name, indentLevel=0): imageFile = f"img/nxdl/nxdl_{name}.png" figure_id = f"fig.nxdl_{name}" - file_name = directories.manual_source_sphinxsroot() / imageFile + file_name = directories.manual_source_sphinxroot() / imageFile if not file_name.exists(): return text = FIGURE_FMT % ( diff --git a/dev_tools/globals/directories.py b/dev_tools/globals/directories.py index a80f2f1ac5..9b8498ef9a 100644 --- a/dev_tools/globals/directories.py +++ b/dev_tools/globals/directories.py @@ -85,28 +85,34 @@ def impatient_build_root() -> Path: return get_build_root() / "impatient-guide" -def manual_source_sphinxsroot() -> Path: +def manual_source_sphinxroot() -> Path: """Sphinx source directory of the NeXus User Manual""" return manual_source_root() / "source" -def manual_build_sphinxsroot() -> Path: +def manual_build_sphinxroot() -> Path: """Sphinx source directory for building of the NeXus User Manual""" return manual_build_root() / "source" def manual_build_staticroot() -> Path: """Static source directory for building of the NeXus User Manual""" - return manual_build_sphinxsroot() / "_static" + return manual_build_sphinxroot() / "_static" def nxclass_build_root(nxdl_file: Path) -> Path: """NeXus class documentation directory for building of the NeXus User Manual""" - root = manual_build_sphinxsroot() / "classes" + root = manual_build_sphinxroot() / "classes" root /= os.path.relpath(nxdl_file.parent, get_nxdl_root()) return root +def get_rst_filename(nxdl_file: Path) -> Path: + rst_file_name = nxclass_build_root(nxdl_file) + rst_file_name /= nxdl_file.with_suffix("").with_suffix(".rst").name + return rst_file_name + + _XSD_FILE_NAME = "nxdl.xsd" _XSD_UNITS_FILE_NAME = "nxdlTypes.xsd" _VERSION_FILE_NAME = "NXDL_VERSION" From 066a7aba53190b76c376e399ed7554b632684976 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 30 Jun 2022 00:53:41 +0200 Subject: [PATCH 5/6] sort keys in JSON anchor file --- dev_tools/docs/anchor_list.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_tools/docs/anchor_list.py b/dev_tools/docs/anchor_list.py index 070927dae9..df668ead01 100644 --- a/dev_tools/docs/anchor_list.py +++ b/dev_tools/docs/anchor_list.py @@ -207,7 +207,7 @@ def _write_json(self, contents): if not self._json_file: return with open(self._json_file, "w") as f: - json.dump(contents, f, indent=4) + json.dump(contents, f, indent=4, sort_keys=True) f.write("\n") def _write_txt(self): From dbe233d37a5247b57e90bd44b2dc36c64887e702 Mon Sep 17 00:00:00 2001 From: woutdenolf Date: Thu, 30 Jun 2022 22:08:16 +0200 Subject: [PATCH 6/6] remove slash from URL --- dev_tools/globals/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_tools/globals/urls.py b/dev_tools/globals/urls.py index 3d70950556..729b7d1028 100644 --- a/dev_tools/globals/urls.py +++ b/dev_tools/globals/urls.py @@ -1,2 +1,2 @@ REPO_URL = "https://github.com/nexusformat/definitions/blob/main" -MANUAL_URL = "https://manual.nexusformat.org/" +MANUAL_URL = "https://manual.nexusformat.org"