Verification

.gitignore

@@ -201,6 +201,8 @@ cython_debug/
 *.txt
 !requirements.txt
 !dev-requirements.txt
+!pynxtools/dataconverter/units/default_en.txt
+!pynxtools/dataconverter/units/constants_en.txt
 !mkdocs-requirements.txt
 !pynxtools/nexus-version.txt
 build/

MANIFEST.in

@@ -7,5 +7,6 @@ recursive-include pynxtools/definitions/base_classes/ *.xml
 recursive-include pynxtools/definitions/applications/ *.xml
 recursive-include pynxtools/definitions/contributed_definitions/ *.xml
 include pynxtools/definitions/*.xsd
+include pynxtools/dataconverter/units *.txt
 include pynxtools/nexus-version.txt
-include pynxtools/definitions/NXDL_VERSION
+include pynxtools/definitions/NXDL_VERSION

dev-requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.11
+# This file is autogenerated by pip-compile with Python 3.10
 # by the following command:
 #
 #    pip-compile --extra=dev --extra=docs --output-file=dev-requirements.txt pyproject.toml
@@ -34,6 +34,8 @@ cycler==0.12.1
     # via matplotlib
 distlib==0.3.8
     # via virtualenv
+exceptiongroup==1.2.1
+    # via pytest
 filelock==3.13.3
     # via virtualenv
 fonttools==4.50.0
@@ -120,6 +122,8 @@ pathspec==0.12.1
     # via mkdocs
 pillow==10.2.0
     # via matplotlib
+pint==0.23
+    # via pynxtools (pyproject.toml)
 pip-tools==7.4.1
     # via pynxtools (pyproject.toml)
 platformdirs==4.2.0
@@ -181,14 +185,24 @@ structlog==24.1.0
     # via pynxtools (pyproject.toml)
 termcolor==2.4.0
     # via mkdocs-macros-plugin
+tomli==2.0.1
+    # via
+    #   build
+    #   coverage
+    #   mypy
+    #   pip-tools
+    #   pyproject-hooks
+    #   pytest
 types-pytz==2024.1.0.20240203
     # via pynxtools (pyproject.toml)
 types-pyyaml==6.0.12.20240311
     # via pynxtools (pyproject.toml)
 types-requests==2.31.0.20240311
     # via pynxtools (pyproject.toml)
 typing-extensions==4.10.0
-    # via mypy
+    # via
+    #   mypy
+    #   pint
 tzdata==2024.1
     # via pandas
 urllib3==2.2.1

pynxtools/dataconverter/convert.py

@@ -37,7 +37,7 @@
 from pynxtools.dataconverter.readers.base.reader import BaseReader
 from pynxtools.dataconverter.template import Template
 from pynxtools.dataconverter.writer import Writer
-from pynxtools.nexus import nexus
+from pynxtools.definitions.dev_tools.utils.nxdl_utils import get_nexus_definitions_path
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -120,7 +120,7 @@ def get_nxdl_root_and_path(nxdl: str):
         Error if no file with the given nxdl name is found.
     """
     # Reading in the NXDL and generating a template
-    definitions_path = nexus.get_nexus_definitions_path()
+    definitions_path = get_nexus_definitions_path()
     if nxdl == "NXtest":
         nxdl_f_path = os.path.join(
             f"{os.path.abspath(os.path.dirname(__file__))}/../../",

pynxtools/dataconverter/helpers.py

@@ -29,12 +29,21 @@
 import lxml.etree as ET
 import numpy as np
 from ase.data import chemical_symbols
+from pint import UndefinedUnitError
 
+import pynxtools.definitions.dev_tools.utils.nxdl_utils as nexus
 from pynxtools import get_nexus_version, get_nexus_version_hash
 from pynxtools.dataconverter.template import Template
-from pynxtools.definitions.dev_tools.utils.nxdl_utils import get_inherited_nodes
-from pynxtools.nexus import nexus
-from pynxtools.nexus.nexus import NxdlAttributeNotFoundError
+from pynxtools.dataconverter.units import ureg
+from pynxtools.definitions.dev_tools.utils.nxdl_utils import (
+    NxdlAttributeNotFoundError,
+    get_enums,
+    get_inherited_nodes,
+    get_node_at_nxdl_path,
+)
+from pynxtools.definitions.dev_tools.utils.nxdl_utils import (
+    get_required_string as nexus_get_required_string,
+)
 
 logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
@@ -50,6 +59,8 @@ class ValidationProblem(Enum):
     InvalidType = 7
     InvalidDatetime = 8
     IsNotPosInt = 9
+    InvalidUnit = 10
+    InvalidTransformationType = 11
 
 
 class Collector:
@@ -109,6 +120,16 @@ def insert_and_log(
             logger.warning(
                 f"The value at {path} should be a positive int, but is {value}."
             )
+        elif log_type == ValidationProblem.InvalidUnit:
+            logger.warning(
+                f"Invalid unit in {path}. {value} "
+                f"is not in unit category {args[0] if args else '<unknown>'}"
+            )
+        elif log_type == ValidationProblem.InvalidTransformationType:
+            logger.warning(
+                f"Invalid transformation type in {path}: {value}. "
+                "Should be either not present or have the value 'translation' or 'rotation'."
+            )
         self.data.add(path)
 
     def has_validation_problems(self):
@@ -195,7 +216,7 @@ def get_all_defined_required_children(nxdl_path, nxdl_name):
     if nxdl_name == "NXtest":
         return []
 
-    elist = nexus.get_inherited_nodes(nxdl_path, nx_name=nxdl_name)[2]
+    elist = get_inherited_nodes(nxdl_path, nx_name=nxdl_name)[2]
     list_of_children_to_add = set()
     for elem in elist:
         list_of_children_to_add.update(get_all_defined_required_children_for_elem(elem))
@@ -298,7 +319,7 @@ def generate_template_from_nxdl(
 
 def get_required_string(elem):
     """Helper function to return nicely formatted names for optionality."""
-    return nexus.get_required_string(elem)[2:-2].lower()
+    return nexus_get_required_string(elem)[2:-2].lower()
 
 
 def convert_nexus_to_caps(nexus_name):
@@ -369,7 +390,7 @@ def convert_data_dict_path_to_hdf5_path(path) -> str:
 def is_value_valid_element_of_enum(value, elist) -> Tuple[bool, list]:
     """Checks whether a value has to be specific from the NXDL enumeration and returns options."""
     for elem in elist:
-        enums = nexus.get_enums(elem)
+        enums = get_enums(elem)
         if enums is not None:
             return value in enums, enums
     return True, []
@@ -448,6 +469,25 @@ def convert_str_to_bool_safe(value):
     return None
 
 
+def clean_str_attr(
+    attr: Optional[Union[str, bytes]], encoding="utf-8"
+) -> Optional[str]:
+    """
+    Cleans the string attribute which means it will decode bytes to str if necessary.
+    If `attr` is not str, bytes or None it raises a TypeError.
+    """
+    if attr is None:
+        return attr
+    if isinstance(attr, bytes):
+        return attr.decode(encoding)
+    if isinstance(attr, str):
+        return attr
+
+    raise TypeError(
+        "Invalid type {type} for attribute. Should be either None, bytes or str."
+    )
+
+
 def is_valid_data_field(value, nxdl_type, path):
     """Checks whether a given value is valid according to what is defined in the NXDL.
 
@@ -487,6 +527,46 @@ def is_valid_data_field(value, nxdl_type, path):
     return value
 
 
+def is_valid_unit(
+    unit: str, nx_category: str, transformation_type: Optional[str]
+) -> bool:
+    """
+    The provided unit belongs to the provided nexus unit category.
+
+    Args:
+        unit (str): The unit to check. Should be according to pint.
+        nx_category (str): A nexus unit category, e.g. `NX_LENGTH`,
+            or derived unit category, e.g., `NX_LENGTH ** 2`.
+        transformation_type (Optional[str]):
+            The transformation type of an NX_TRANSFORMATION.
+            This parameter is ignored if the `nx_category` is not `NX_TRANSFORMATION`.
+            If `transformation_type` is not present this should be set to None.
+
+    Returns:
+        bool: The unit belongs to the provided category
+    """
+    unit = clean_str_attr(unit)
+    try:
+        if nx_category in ("NX_ANY"):
+            ureg(unit)  # Check if unit is generally valid
+            return True
+        nx_category = re.sub(r"(NX_[A-Z]+)", r"[\1]", nx_category)
+        if nx_category == "[NX_TRANSFORMATION]":
+            # NX_TRANSFORMATIONS is a pseudo unit
+            # and can be either an angle, a length or unitless
+            # depending on the transformation type.
+            if transformation_type is None:
+                return ureg(unit).check("[NX_UNITLESS]")
+            if transformation_type == "translation":
+                return ureg(unit).check("[NX_LENGTH]")
+            if transformation_type == "rotation":
+                return ureg(unit).check("[NX_ANGLE]")
+            return False
+        return ureg(unit).check(f"{nx_category}")
+    except UndefinedUnitError:
+        return False
+
+
 @lru_cache(maxsize=None)
 def path_in_data_dict(nxdl_path: str, data_keys: Tuple[str, ...]) -> List[str]:
     """Checks if there is an accepted variation of path in the dictionary & returns the path."""
@@ -505,9 +585,9 @@ def check_for_optional_parent(path: str, nxdl_root: ET.Element) -> str:
         return "<<NOT_FOUND>>"
 
     parent_nxdl_path = convert_data_converter_dict_to_nxdl_path(parent_path)
-    elem = nexus.get_node_at_nxdl_path(nxdl_path=parent_nxdl_path, elem=nxdl_root)
+    elem = get_node_at_nxdl_path(nxdl_path=parent_nxdl_path, elem=nxdl_root)
 
-    if nexus.get_required_string(elem) in ("<<OPTIONAL>>", "<<RECOMMENDED>>"):
+    if get_required_string(elem) in ("<<OPTIONAL>>", "<<RECOMMENDED>>"):
         return parent_path
 
     return check_for_optional_parent(parent_path, nxdl_root)
@@ -522,8 +602,8 @@ def is_node_required(nxdl_key, nxdl_root):
             nxdl_key[0 : nxdl_key.rindex("/") + 1]
             + nxdl_key[nxdl_key.rindex("/") + 2 :]
         )
-    node = nexus.get_node_at_nxdl_path(nxdl_key, elem=nxdl_root, exc=False)
-    return nexus.get_required_string(node) == "<<REQUIRED>>"
+    node = get_node_at_nxdl_path(nxdl_key, elem=nxdl_root, exc=False)
+    return get_required_string(node) == "<<REQUIRED>>"
 
 
 def all_required_children_are_set(optional_parent_path, data, nxdl_root):
@@ -753,7 +833,7 @@ def try_undocumented(data, nxdl_root: ET.Element):
             field_path = path.rsplit("/", 1)[0]
             if field_path in data.get_documented() and path in data.undocumented:
                 field_requiredness = get_required_string(
-                    nexus.get_node_at_nxdl_path(
+                    get_node_at_nxdl_path(
                         nxdl_path=convert_data_converter_dict_to_nxdl_path(field_path),
                         elem=nxdl_root,
                     )
@@ -767,7 +847,7 @@ def try_undocumented(data, nxdl_root: ET.Element):
             nxdl_path = nxdl_path[0:index_of_at] + nxdl_path[index_of_at + 1 :]
 
         try:
-            elem = nexus.get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
+            elem = get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
             optionality = get_required_string(elem)
             data[optionality][path] = data.undocumented[path]
             del data.undocumented[path]
@@ -786,7 +866,7 @@ def validate_data_dict(template, data, nxdl_root: ET.Element):
 
     @lru_cache(maxsize=None)
     def get_xml_node(nxdl_path: str) -> ET.Element:
-        return nexus.get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
+        return get_node_at_nxdl_path(nxdl_path=nxdl_path, elem=nxdl_root)
 
     # Make sure all required fields exist.
     ensure_all_required_fields_exist(template, data, nxdl_root)
@@ -814,21 +894,33 @@ def get_xml_node(nxdl_path: str) -> ET.Element:
                     )
                     continue
 
-                # TODO: If we want we could also enable unit validation here
-                # field = nexus.get_node_at_nxdl_path(
-                #     nxdl_path=convert_data_converter_dict_to_nxdl_path(
-                #         # The part below is the backwards compatible version of
-                #         # nxdl_path.removesuffix("/units")
-                #         nxdl_path[:-6] if nxdl_path.endswith("/units") else nxdl_path
-                #     ),
-                #     elem=nxdl_root,
-                # )
-                # nxdl_unit = field.attrib.get("units", "")
-                # if not is_valid_unit(data[path], nxdl_unit):
-                #     raise ValueError(
-                #         f"Invalid unit in {path}. {data[path]} "
-                #         f"is not in unit category {nxdl_unit}"
-                #     )
+                field = get_node_at_nxdl_path(
+                    nxdl_path=convert_data_converter_dict_to_nxdl_path(
+                        # The part below is the backwards compatible version of
+                        # nxdl_path.removesuffix("/units")
+                        nxdl_path[:-6] if nxdl_path.endswith("/units") else nxdl_path
+                    ),
+                    elem=nxdl_root,
+                )
+                nxdl_unit = field.attrib.get("units", "")
+                transformation_type = (
+                    field.attrib.get("transformation_type")
+                    if nxdl_unit == "[NX_TRANSFORMATION]"
+                    else None
+                )
+                if not is_valid_unit(data[path], nxdl_unit, transformation_type):
+                    if transformation_type is not None and transformation_type not in (
+                        "rotation",
+                        "translation",
+                    ):
+                        collector.insert_and_log(
+                            path,
+                            ValidationProblem.InvalidTransformationType,
+                            transformation_type,
+                        )
+                    collector.insert_and_log(
+                        path, ValidationProblem.InvalidUnit, data[path], nxdl_unit
+                    )
                 continue
 
             elem = get_xml_node(nxdl_path)
@@ -851,7 +943,7 @@ def get_xml_node(nxdl_path: str) -> ET.Element:
                     else "NXDL_TYPE_UNAVAILABLE"
                 )
                 data[path] = is_valid_data_field(data[path], nxdl_type, path)
-                elist = nexus.get_inherited_nodes(
+                elist = get_inherited_nodes(
                     nxdl_path, path.rsplit("/", 1)[-1], nxdl_root
                 )[2]
                 is_valid_enum, enums = is_value_valid_element_of_enum(data[path], elist)
@@ -934,6 +1026,7 @@ def update_and_warn(key: str, value: str):
         f"blob/{get_nexus_version_hash()}",
     )
     update_and_warn("/@NeXus_version", get_nexus_version())
+    # pylint: disable=c-extension-no-member
     update_and_warn("/@HDF5_version", ".".join(map(str, h5py.h5.get_libversion())))
     update_and_warn("/@h5py_version", h5py.__version__)
 

pynxtools/dataconverter/readers/ellips/reader.py

@@ -450,6 +450,7 @@ def read(
                 # MK:: Carola, Ron, Flo, Tamas, Sandor refactor the following line
                 template[f"/ENTRY[entry]/plot/DATA[{key}_errors]/@units"] = "degree"
 
+        template["/ENTRY[entry]/data_collection/measured_data/@units"] = ""
         # Define default plot showing Psi and Delta at all angles:
         template["/@default"] = "entry"
         template["/ENTRY[entry]/@default"] = "plot"

pynxtools/dataconverter/readers/json_map/README.md

@@ -39,7 +39,7 @@ Example:
 
 ```json
   "/ENTRY[entry]/DATA[data]/current_295C": "/entry/data/current_295C",
-  "/ENTRY[entry]/NXODD_name/posint_value": "/a_level_down/another_level_down/posint_value",
+  "/ENTRY[entry]/NXODD_name[odd_name]/posint_value": "/a_level_down/another_level_down/posint_value",
 ```
 
 * Write the values directly in the mapping file for missing data from your data file.

pynxtools/dataconverter/units/__init__.py

@@ -0,0 +1,23 @@
+#
+# Copyright The NOMAD Authors.
+#
+# This file is part of NOMAD. See https://nomad-lab.eu for further info.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""A unit registry for nexus units"""
+
+import os
+from pint import UnitRegistry
+
+ureg = UnitRegistry(os.path.join(os.path.dirname(__file__), "default_en.txt"))