Merge branch 'main' into v2

docs/conformance.rst

@@ -1,17 +1,23 @@
 Conformance
 ===========
 
-``ocfl-py`` is designed to follow closely the OCFL specifications.
+``ocfl-py`` is designed to follow closely the OCFL specifications, which
+includes reporting errors in response to violations of MUST and MUST NOT
+requirements in the specification, and warnings in response to not following
+SHOULD or SHOULD NOT requirements in the specification (see the `Conformance
+<https://ocfl.io/1.1/spec/#conformance>`_ section).
 
 The following is list of error and warning codes implemented but the ``ocfl-py``
-package. The complete list `OCFL Validation Codes
-<https://ocfl.io/1.1/spec/validation-codes.html>`_ is provided as part of the
-OCFL specifications.
+package:
 
 .. toctree::
    :maxdepth: 1
 
    validation_status
 
-(The validation status table is generated by the `ocfl-validate.py` script
+The complete list `OCFL Validation Codes
+<https://ocfl.io/1.1/spec/validation-codes.html>`_ is provided as part of the
+OCFL specifications.
+
+(The validation status table is generated by the `extract_codes.py` script
 run against the ``ocfl`` module.)

docs/demo_build_v1_0_spec_examples.md

@@ -52,34 +52,8 @@ This is inventory should match the example with 3 versions in <https://ocfl.io/1
 {
   "digestAlgorithm": "sha512-spec-ex",
   "fixity": {
-    "md5": {
-      "184f84e28cbe75e050e9c25ea7f2e939": [
-        "v1/content/foo/bar.xml"
-      ],
-      "2673a7b11a70bc7ff960ad8127b4adeb": [
-        "v2/content/foo/bar.xml"
-      ],
-      "c289c8ccd4bab6e385f5afdd89b5bda2": [
-        "v1/content/image.tiff"
-      ],
-      "d41d8cd98f00b204e9800998ecf8427e": [
-        "v1/content/empty.txt"
-      ]
-    },
-    "sha1": {
-      "66709b068a2faead97113559db78ccd44712cbf2": [
-        "v1/content/foo/bar.xml"
-      ],
-      "a6357c99ecc5752931e133227581e914968f3b9c": [
-        "v2/content/foo/bar.xml"
-      ],
-      "b9c7ccc6154974288132b63c15db8d2750716b49": [
-        "v1/content/image.tiff"
-      ],
-      "da39a3ee5e6b4b0d3255bfef95601890afd80709": [
-        "v1/content/empty.txt"
-      ]
-    }
+    "md5": {},
+    "sha1": {}
   },
   "head": "v3",
   "id": "ark:/12345/bcd987",
@@ -97,7 +71,7 @@ This is inventory should match the example with 3 versions in <https://ocfl.io/1
       "v1/content/image.tiff"
     ]
   },
-  "type": "https://ocfl.io/1.0/spec/#inventory",
+  "type": "https://ocfl.io/1.1/spec/#inventory",
   "versions": {
     "v1": {
       "created": "2018-01-01T01:01:01Z",

docs/demo_build_v1_1_spec_examples.md

@@ -52,34 +52,8 @@ This is inventory should match the example with 3 versions in <https://ocfl.io/1
 {
   "digestAlgorithm": "sha512-spec-ex",
   "fixity": {
-    "md5": {
-      "184f84e28cbe75e050e9c25ea7f2e939": [
-        "v1/content/foo/bar.xml"
-      ],
-      "2673a7b11a70bc7ff960ad8127b4adeb": [
-        "v2/content/foo/bar.xml"
-      ],
-      "c289c8ccd4bab6e385f5afdd89b5bda2": [
-        "v1/content/image.tiff"
-      ],
-      "d41d8cd98f00b204e9800998ecf8427e": [
-        "v1/content/empty.txt"
-      ]
-    },
-    "sha1": {
-      "66709b068a2faead97113559db78ccd44712cbf2": [
-        "v1/content/foo/bar.xml"
-      ],
-      "a6357c99ecc5752931e133227581e914968f3b9c": [
-        "v2/content/foo/bar.xml"
-      ],
-      "b9c7ccc6154974288132b63c15db8d2750716b49": [
-        "v1/content/image.tiff"
-      ],
-      "da39a3ee5e6b4b0d3255bfef95601890afd80709": [
-        "v1/content/empty.txt"
-      ]
-    }
+    "md5": {},
+    "sha1": {}
   },
   "head": "v3",
   "id": "ark:/12345/bcd987",

docs/demo_using_bagit_bags.md

@@ -62,8 +62,8 @@ INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/t
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v2/bagit.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v2/bag-info.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v2/manifest-sha512.txt
-INFO:root:Updated OCFL object info:bb123cd4567 in tmp/obj by adding v2
-### <ocfl.version_metadata.VersionMetadata object at 0x7fb92132e6e0>
+INFO:root:Updated OCFL object info:bb123cd4567 by adding v2
+### <ocfl.version_metadata.VersionMetadata object at 0x7f1cf4eaa5f0>
 Updated object info:bb123cd4567 to v2
 ```
 
@@ -103,8 +103,8 @@ INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/t
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v3/bagit.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v3/bag-info.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v3/manifest-sha512.txt
-INFO:root:Updated OCFL object info:bb123cd4567 in tmp/obj by adding v3
-### <ocfl.version_metadata.VersionMetadata object at 0x7f1bba34a6e0>
+INFO:root:Updated OCFL object info:bb123cd4567 by adding v3
+### <ocfl.version_metadata.VersionMetadata object at 0x7fb59dd525f0>
 Updated object info:bb123cd4567 to v3
 ```
 
@@ -149,8 +149,8 @@ INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/t
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v4/bagit.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v4/bag-info.txt
 INFO:bagit:Verifying checksum for file /home/runner/work/ocfl-py/ocfl-py/tests/testdata/bags/uaa_v4/manifest-sha512.txt
-INFO:root:Updated OCFL object info:bb123cd4567 in tmp/obj by adding v4
-### <ocfl.version_metadata.VersionMetadata object at 0x7f37a7dfe6e0>
+INFO:root:Updated OCFL object info:bb123cd4567 by adding v4
+### <ocfl.version_metadata.VersionMetadata object at 0x7f924be4a5f0>
 Updated object info:bb123cd4567 to v4
 ```
 
@@ -164,12 +164,13 @@ Taking the newly created OCFL object `/tmp/obj` we can `--extract` the `v4` cont
 INFO:root:Extracted v4 into tmp/extracted_v4
 INFO:bagit:Creating bag for directory tmp/extracted_v4
 INFO:bagit:Creating data directory
-INFO:bagit:Moving my_content to tmp/extracted_v4/tmpt_h62tn4/my_content
-INFO:bagit:Moving tmp/extracted_v4/tmpt_h62tn4 to data
+INFO:bagit:Moving my_content to tmp/extracted_v4/tmp6gveuavx/my_content
+INFO:bagit:Moving tmp/extracted_v4/tmp6gveuavx to data
 INFO:bagit:Using 1 processes to generate manifests: sha512
 INFO:bagit:Generating manifest lines for file data/my_content/dracula.txt
 INFO:bagit:Generating manifest lines for file data/my_content/dunwich.txt
 INFO:bagit:Generating manifest lines for file data/my_content/poe-nevermore.txt
+INFO:bagit:Generating manifest lines for file data/my_content/another_directory/a_third_copy_of_dracula.txt
 INFO:bagit:Creating bagit.txt
 INFO:bagit:Creating bag-info.txt
 INFO:bagit:Creating tmp/extracted_v4/tagmanifest-sha512.txt
@@ -186,24 +187,14 @@ We note that the OCFL object had only one `content` file in `v4` but the extract
 diff -r tmp/extracted_v4/bag-info.txt tests/testdata/bags/uaa_v4/bag-info.txt
 1,2c1
 < Bag-Software-Agent: bagit.py v1.8.1 <https://github.com/LibraryOfCongress/bagit-python>
-< Bagging-Date: 2024-12-05
+< Bagging-Date: 2024-12-06
 ---
 > Bagging-Date: 2020-01-04
-7c6
-< Payload-Oxum: 1032810.3
----
-> Payload-Oxum: 1915970.4
-Only in tests/testdata/bags/uaa_v4/data/my_content: another_directory
-diff -r tmp/extracted_v4/manifest-sha512.txt tests/testdata/bags/uaa_v4/manifest-sha512.txt
-3a4
-> ffc150e7944b5cf5ddb899b2f48efffbd490f97632fc258434aefc4afb92aef2e3441ddcceae11404e5805e1b6c804083c9398c28f061c9ba42dd4bac53d5a2e  data/my_content/another_directory/a_third_copy_of_dracula.txt
 diff -r tmp/extracted_v4/tagmanifest-sha512.txt tests/testdata/bags/uaa_v4/tagmanifest-sha512.txt
-2,3c2,3
-< ba0f77d695226f32ef316aa1c0000a13d8f78b6948a5415a3e11a95f73153ecf351c91211a5e7b20ab6b785ec472ec5fcc6e8e6fb5a3e718ea86d7719cd3c690 bag-info.txt
-< e4cb52067909ba58668f21c0da72c73c586a3f87d64b471187cb6234ed605d17974e4c7833f6ae7e639ab1126502534b3bed4cfeee8cba485cfc764b48225e0f manifest-sha512.txt
+2c2
+< 7e23b308ac51b064e7471d7b8e5ba1f758891631ad8c8fb57799a39018d7d77e893a8236a608a8087117000c55efde9529cb76cdb63bacc5642b38ab459b30d5 bag-info.txt
 ---
 > 10624e6d45462def7af66d1a0d977606c7b073b01809c1d42258cfab5c34a275480943cbe78044416aee1f23822cc3762f92247b8f39b5c6ddc5ae32a8f94ce5 bag-info.txt
-> 5c2e2b9cacc93cb315d57f09fac6d199c3378313b6cf918bb0a70e1839c4e4c0c2e5a7f9ae869cf7755e09a196a835be1af7c510d3d5faa5d0c0b3f6be9f816a manifest-sha512.txt
 ```
 
 (last command exited with return code 1)

docs/fixtures.rst

@@ -8,21 +8,21 @@ Official `OCFL Fixtures
 
   * Specification examples:
 
-    * Section `5.1 Minimal OCFL Object <https://ocfl.io/1.1/spec/#example-minimal-object>`_ is good
+    * Section `5.1 Minimal OCFL v1.1 Object <https://ocfl.io/1.1/spec/#example-minimal-object>`_ is good
 
-    * Section `5.2 Versioned OCFL Object <https://ocfl.io/1.1/spec/#example-versioned-object>`_ is valid and has fixture object `1.1/good-objects/spec-ex-full <https://github.com/OCFL/fixtures/tree/main/1.1/good-objects/spec-ex-full>`_
+    * Section `5.2 Versioned OCFL v1.1 Object <https://ocfl.io/1.1/spec/#example-versioned-object>`_ is valid and has fixture object `1.1/good-objects/spec-ex-full <https://github.com/OCFL/fixtures/tree/main/1.1/good-objects/spec-ex-full>`_
 
-    * Section `5.3 Different Logical and Content Paths in an OCFL Object <https://ocfl.io/1.1/spec/#example-object-diff-paths>`_ is valid but should generate a warning because the ``message`` property is missing.
+    * Section `5.3 Different Logical and Content Paths in an OCFL v1.1 Object <https://ocfl.io/1.1/spec/#example-object-diff-paths>`_ is valid but should generate a warning because the ``message`` property is missing.
 
 * OCFL v1.0
 
   * Specification examples:
 
-    * Section `5.1 Minimal OCFL Object <https://ocfl.io/1.0/spec/#example-minimal-object>`_ is valid but should generate a warning because the ``address`` is not a URI
+    * Section `5.1 Minimal OCFL v1.0 Object <https://ocfl.io/1.0/spec/#example-minimal-object>`_ is valid but should generate a warning because the ``address`` is not a URI
 
-    * Section `5.2 Versioned OCFL Object <https://ocfl.io/1.0/spec/#example-versioned-object>`_ is valid and has fixture object `1.0/good-objects/spec-ex-full <https://github.com/OCFL/fixtures/tree/main/1.0/good-objects/spec-ex-full>`_
+    * Section `5.2 Versioned OCFL v1.0 Object <https://ocfl.io/1.0/spec/#example-versioned-object>`_ is valid and has fixture object `1.0/good-objects/spec-ex-full <https://github.com/OCFL/fixtures/tree/main/1.0/good-objects/spec-ex-full>`_
 
-    * Section `5.3 Different Logical and Content Paths in an OCFL Object <https://ocfl.io/1.0/spec/#example-object-diff-paths>`_ is valid but should generate warnings because the ``message`` and the ``user`` ``name`` and ``address`` properties are missing.
+    * Section `5.3 Different Logical and Content Paths in an OCFL v1.0 Object <https://ocfl.io/1.0/spec/#example-object-diff-paths>`_ is valid but should generate warnings because the ``message`` and the ``user`` ``name`` and ``address`` properties are missing.
 
-Additions from `ocfl-py
-<https://github.com/zimeon/ocfl-py/tree/main/extra_fixtures>`_ are in the ``extra_fixtures`` directory
+Additions from ``ocfl-py`` are in the `extra_fixtures
+<https://github.com/zimeon/ocfl-py/tree/main/extra_fixtures>`_  directory.

docs/validation_status.md

@@ -53,7 +53,7 @@ The following tables show the implementation status of all errors and warnings i
 | [E030](https://ocfl.io/1.1/spec#E030) | 'SHA-256 algorithm defined by [FIPS-180-4] and must be encoded using hex (base16) encoding [RFC4648].' | _Not implemented_ |
 | [E031](https://ocfl.io/1.1/spec#E031) | 'SHA-512 algorithm defined by [FIPS-180-4] and must be encoded using hex (base16) encoding [RFC4648].' | _Not implemented_ |
 | [E032](https://ocfl.io/1.1/spec#E032) | '[blake2b-512] must be encoded using hex (base16) encoding [RFC4648].' | _Not implemented_ |
-| [E033](https://ocfl.io/1.1/spec#E033) | 'An OCFL Object Inventory MUST follow the [JSON] structure described in this section and must be named inventory.json.' | OCFL Object %s inventory is not valid JSON (%s) \[[ocfl/object.py#L738](https://github.com/zimeon/ocfl-py/blob/main/ocfl/object.py#L738) [ocfl/validator.py#L201](https://github.com/zimeon/ocfl-py/blob/main/ocfl/validator.py#L201)\] |
+| [E033](https://ocfl.io/1.1/spec#E033) | 'An OCFL Object Inventory MUST follow the [JSON] structure described in this section and must be named inventory.json.' | OCFL Object %s inventory is not valid JSON (%s) \[[ocfl/object.py#L643](https://github.com/zimeon/ocfl-py/blob/main/ocfl/object.py#L643) [ocfl/validator.py#L201](https://github.com/zimeon/ocfl-py/blob/main/ocfl/validator.py#L201)\] |
 | [E034](https://ocfl.io/1.1/spec#E034) | 'An OCFL Object Inventory must follow the [JSON] structure described in this section and MUST be named inventory.json.' | _Not implemented_ |
 | [E035](https://ocfl.io/1.1/spec#E035) | 'The forward slash (/) path separator must be used in content paths in the manifest and fixity blocks within the inventory.' | _Not implemented_ |
 | [E036](https://ocfl.io/1.1/spec#E036) | 'An OCFL Object Inventory must include the following keys: [id, type, digestAlgorithm, head]' | _See multiple cases identified with suffixes below_ |
@@ -206,4 +206,4 @@ The following tables show the implementation status of all errors and warnings i
 | [W016](https://ocfl.io/1.1/spec#W016) | 'In the Storage Root, extension sub-directories SHOULD be named according to a registered extension name.' | _Not implemented_ |
 | W901 | **Not in specification** | OCFL Storage Root includes unregistered extension directory '%s' \[[ocfl/storage_root.py#L275](https://github.com/zimeon/ocfl-py/blob/main/ocfl/storage_root.py#L275)\] |
 
-_Generated by `extract_codes.py` at 2024-12-05 18:22:36.654306_
+_Generated by `extract_codes.py` at 2024-12-06 00:46:27.116525_

extra_fixtures/1.1/bad-objects/E038_type_not_str/0=ocfl_object_1.1

@@ -0,0 +1 @@
+ocfl_object_1.1

extra_fixtures/1.1/bad-objects/E038_type_not_str/inventory.json

@@ -0,0 +1,15 @@
+{
+  "digestAlgorithm": "sha512",
+  "head": "v1",
+  "id": "http://example.org/E038_type_not_str",
+  "manifest": { },
+  "type": [ "THIS IS AN ARRAY NOT JUST A STRING" ],
+  "versions": {
+    "v1": {
+      "created": "2019-01-01T02:03:04Z",
+      "message": "One version and no content",
+      "state": { },
+      "user": { "address": "mailto:[email protected]", "name": "Person A" }
+    }
+  }
+}

extra_fixtures/1.1/bad-objects/E038_type_not_str/inventory.json.sha512

@@ -0,0 +1 @@
+6cdd5082fe06fa39d31627c05c09dcab4e32dc4d00caaf4c436129402b37daae1e938b3e0e3ee1b02dd09764fd3fd7e4ffa918657f4df57a7da879382df537f8 inventory.json

extra_fixtures/1.1/bad-objects/E038_type_not_str/v1/inventory.json

@@ -0,0 +1,15 @@
+{
+  "digestAlgorithm": "sha512",
+  "head": "v1",
+  "id": "http://example.org/E038_type_not_str",
+  "manifest": { },
+  "type": [ "THIS IS AN ARRAY NOT JUST A STRING" ],
+  "versions": {
+    "v1": {
+      "created": "2019-01-01T02:03:04Z",
+      "message": "One version and no content",
+      "state": { },
+      "user": { "address": "mailto:[email protected]", "name": "Person A" }
+    }
+  }
+}

extra_fixtures/1.1/bad-objects/E038_type_not_str/v1/inventory.json.sha512

@@ -0,0 +1 @@
+6cdd5082fe06fa39d31627c05c09dcab4e32dc4d00caaf4c436129402b37daae1e938b3e0e3ee1b02dd09764fd3fd7e4ffa918657f4df57a7da879382df537f8 inventory.json

ocfl/data/validation-errors.json

@@ -220,6 +220,12 @@
        "en": "OCFL Object %s inventory `type` attribute has an unsupported specification version number (%s), will proceed as if using version %s"
     }
   },
+  "E038d": {
+    "params": ["where"],
+    "description": {
+       "en": "OCFL Object %s inventory `type` attribute does not have a string value"
+    }
+  },
   "E040": {
     "params": ["where", "got", "expected"],
     "description": {

ocfl/inventory.py

@@ -1,5 +1,11 @@
 """OCFL Inventory and Version.
 
+The Inventory class provides storage for inventory data and mehtods to
+conveniently access and manipulate it. The associated Version class provides
+methods to access and manipulate information about a specific object version.
+Neither of these classes interact with object content, see ocfl.NewVersion
+and ocfl.Object.
+
 The storage mechanism for the inventory data is the python dict()
 structure resulting from reading the inventory JSON file and suitable
 for writing an inventory JSON file. Here we provide convenient
@@ -91,7 +97,11 @@ class Inventory():  # pylint: disable=too-many-public-methods
     attribute is a string that is not set in the underlying data, else
     the value if it is. In the cases that the normal return value would
     be an array or a dict, then and empty array or empty dict are returned
-    if not present in the underlying data.
+    if not present in the underlying data. In some cases, additional methods
+    with a suffix ``_add_if_not_present`` are provided so that assignements
+    will work for previously missing attributes. See, for example, the
+    ``manifest`` property and the corresponding
+    ``manifest_add_if_not_present()`` method.
 
     Attributes:
         data: dict that is the top level JSON object of the parsed JSON
@@ -410,7 +420,7 @@ def add_version(self, vdir=None, metadata=None, state=None,
         self.head = vdir
         return self.version(vdir)
 
-    def add_file(self, *, digest, content_path):
+    def add_file_to_manifest(self, *, digest, content_path):
         """Add file to the manifest.
 
         Arguments:
@@ -559,9 +569,9 @@ def normalize_digests(self, digest_algorithm=None):
 class Version():
     """Version class to represent version information in an Inventory.
 
-    The class stores only pointers to the appropriate inventory and
-    version directory with the versions block. These are used to access
-    data in the inventory.
+    The class stores only pointers to the appropriate ocfl.Inventory object
+    and the version directory key within the versions block. These are used
+    to access and manipulate data for a specific version in the inventory.
     """
 
     def __init__(self, inv, vdir):
@@ -733,7 +743,7 @@ def add_file(self, *, digest, logical_path, content_path=None, dedupe=True):
             content_path = make_unused_filepath(filepath=suggested,
                                                 used=self.inv.content_paths)
             # Have location now, add to manifest
-            self.inv.add_file(digest=digest, content_path=content_path)
+            self.inv.add_file_to_manifest(digest=digest, content_path=content_path)
         else:
             # File or files with same digest exist
             content_path = None

ocfl/inventory_validator.py

@@ -138,7 +138,7 @@ def validate(self, inventory, force_spec_version=None):
         if "type" not in inventory:
             self._error("E036b")
         elif not isinstance(inventory["type"], str):
-            self._error("E999")
+            self._error("E038d")
         elif (force_spec_version
                 and inventory["type"] != "https://ocfl.io/" + force_spec_version + "/spec/#inventory"):
             self._error("E038a", expected="https://ocfl.io/" + force_spec_version + "/spec/#inventory", got=inventory["type"])