support order with cfg.track_order

h5pyd/_hl/base.py

@@ -14,6 +14,7 @@
 
 import posixpath
 import os
+import sys
 import json
 import numpy as np
 import logging
@@ -28,6 +29,10 @@
 numpy_float_types = (np.float16, np.float32, np.float64)
 
 
+def eprint(*args, **kwargs):
+    print(*args, file=sys.stderr, **kwargs)
+
+
 class FakeLock():
     def __init__(self):
         pass
@@ -506,7 +511,7 @@ def readElement(buffer, offset, arr, index, dt):
             e = np.frombuffer(bytes(e_buffer), dtype=dt)
             arr[index] = e[0]
         except ValueError:
-            print(f"ERROR: ValueError setting {e_buffer} and dtype: {dt}")
+            eprint(f"ERROR: ValueError setting {e_buffer} and dtype: {dt}")
             raise
     else:
         # variable length element
@@ -533,7 +538,7 @@ def readElement(buffer, offset, arr, index, dt):
                 try:
                     e = np.frombuffer(bytes(e_buffer), dtype=vlen)
                 except ValueError:
-                    print("ValueError -- e_buffer:", e_buffer, "dtype:", vlen)
+                    eprint("ValueError -- e_buffer:", e_buffer, "dtype:", vlen)
                     raise
                 arr[index] = e
 

h5pyd/_hl/files.py

@@ -327,7 +327,6 @@ def __init__(
         timeout
             Timeout value in seconds
         """
-
         groupid = None
         dn_ids = []
         # if we're passed a GroupId as domain, just initialize the file object
@@ -407,9 +406,6 @@ def __init__(
             if swmr:
                 use_cache = False  # disable metadata caching in swmr mode
 
-            if track_order is None:
-                track_order = cfg.track_order
-
             http_conn = HttpConn(
                 domain,
                 endpoint=endpoint,
@@ -489,7 +485,7 @@ def __init__(
                     body["owner"] = owner
                 if linked_domain:
                     body["linked_domain"] = linked_domain
-                if track_order:
+                if track_order or cfg.track_order:
                     create_props = {"CreateOrder": 1}
                     group_body = {"creationProperties": create_props}
                     body["group"] = group_body
@@ -558,22 +554,20 @@ def __init__(
 
             groupid = GroupID(None, group_json, http_conn=http_conn)
         # end else
+
         self._name = "/"
         self._id = groupid
-        self._verboseInfo = None  # aditional state we'll get when requested
+        self._verboseInfo = None  # additional state we'll get when requested
         self._verboseUpdated = None  # when the verbose data was fetched
         self._lastScan = None  # when summary stats where last updated by server
         self._dn_ids = dn_ids
-        self._track_order = track_order
         self._swmr_mode = swmr
 
         Group.__init__(self, self._id, track_order=track_order)
 
     def _getVerboseInfo(self):
         now = time.time()
-        if (
-            self._verboseUpdated is None or now - self._verboseUpdated > VERBOSE_REFRESH_TIME
-        ):
+        if (self._verboseUpdated is None or now - self._verboseUpdated > VERBOSE_REFRESH_TIME):
             # resynch the verbose data
             req = "/?verbose=1"
             rsp_json = self.GET(req, use_cache=False, params={"CreateOrder": "1" if self._track_order else "0"})

h5pyd/_hl/group.py

@@ -200,6 +200,7 @@ def _make_group(self, parent_id=None, parent_name=None, link=None, track_order=N
         """ helper function to make a group """
 
         cfg = config.get_config()
+
         link_json = {}
         if parent_id:
             link_json["id"] = parent_id
@@ -219,6 +220,8 @@ def _make_group(self, parent_id=None, parent_name=None, link=None, track_order=N
         group_json = rsp
         groupId = GroupID(self, group_json)
         sub_group = Group(groupId)
+        if track_order or cfg.track_order:
+            sub_group._track_order = True
         if parent_name:
             if parent_name[-1] == '/':
                 parent_name = parent_name + link
@@ -272,8 +275,12 @@ def create_group(self, h5path, track_order=None):
                 create_group = True
 
             if create_group:
-                sub_group = self._make_group(parent_id=parent_uuid, parent_name=parent_name, link=link)
-                sub_group._track_order = track_order
+                kwargs = {}
+                kwargs["parent_id"] = parent_uuid
+                kwargs["parent_name"] = parent_name
+                kwargs["link"] = link
+                kwargs["track_order"] = track_order
+                sub_group = self._make_group(**kwargs)
                 parent_uuid = sub_group.id.id
 
             else:
@@ -593,7 +600,7 @@ def __getitem__(self, name, track_order=None):
         # convert bytes to str for PY3
         if isinstance(name, bytes):
             name = name.decode('utf-8')
-        self.log.debug(f"group.__getitem__({name})")
+        self.log.debug(f"group.__getitem__({name}, track_order={track_order})")
 
         tgt = None
         if isinstance(name, h5type.Reference):
@@ -716,7 +723,7 @@ def get(self, name, default=None, getclass=False, getlink=False, track_order=Non
         """
         if not (getclass or getlink):
             try:
-                return self.__getitem__(name, track_order)
+                return self.__getitem__(name, track_order=track_order)
             except KeyError:
                 return default
 

test/hl/test_attribute.py

@@ -292,49 +292,68 @@ def test_delete_multiple(self):
 
 class TestTrackOrder(TestCase):
 
-    def fill_attrs(self, track_order):
-        attrs = self.f.create_group('test', track_order=track_order).attrs
-        for i in range(100):
-            attrs[str(i)] = i
-        return attrs
+    titles = ("one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten")
+
+    def fill_attrs(self, obj):
+        count = len(self.titles)
+        attrs = obj.attrs
+        for i in range(count):
+            title = self.titles[i]
+            val = i + 1
+            attrs[title] = val
 
-    # https://forum.hdfgroup.org/t/bug-h5arename-fails-unexpectedly/4881
     def test_track_order(self):
         filename = self.getFileName("test_test_track_order_attribute")
         print(f"filename: {filename}")
-        self.f = h5py.File(filename, 'w')
-        attrs = self.fill_attrs(track_order=True)  # creation order
-        self.assertEqual(list(attrs),
-                         [str(i) for i in range(100)])
+        with h5py.File(filename, 'w') as f:
+            g1 = f.create_group('test', track_order=True)
+            self.fill_attrs(g1)
+            self.assertEqual(list(g1.attrs), list(self.titles))
+        # group should return track order
+        with h5py.File(filename) as f:
+            g1 = f['test']
+            self.assertEqual(list(g1.attrs), list(self.titles))
+
+    def test_track_order_cfg(self):
+        filename = self.getFileName("test_test_track_order_attribute")
+        print(f"filename: {filename}")
+        cfg = h5py.get_config()
+        with h5py.File(filename, 'w') as f:
+            cfg.track_order = True
+            g1 = f.create_group('test')
+            cfg.track_order = False  # reset
+
+            self.fill_attrs(g1)
+            self.assertEqual(list(g1.attrs), list(self.titles))
+
+        with h5py.File(filename) as f:
+            g1 = f['test']
+            self.assertEqual(list(g1.attrs), list(self.titles))
 
     def test_no_track_order(self):
         filename = self.getFileName("test_test_no_track_order_attribute")
         print(f"filename: {filename}")
-        self.f = h5py.File(filename, 'w')
-        attrs = self.fill_attrs(track_order=False)  # name alphanumeric
-        self.assertEqual(list(attrs),
-                         sorted([str(i) for i in range(100)]))
-
-    def fill_attrs2(self, track_order):
-        group = self.f.create_group('test', track_order=track_order)
-        for i in range(12):
-            group.attrs[str(i)] = i
-        return group
+        f = h5py.File(filename, 'w')
+        g1 = f.create_group('test')  # name alphanumeric
+        self.fill_attrs(g1)
+        self.assertEqual(list(g1.attrs), sorted(list(self.titles)))
 
     def test_track_order_overwrite_delete(self):
         filename = self.getFileName("test_test_track_order_overwrite_delete")
         print(f"filename: {filename}")
-        self.f = h5py.File(filename, 'w')
-        # issue h5py#1385
-        group = self.fill_attrs2(track_order=True)  # creation order
-        self.assertEqual(group.attrs["11"], 11)
+        f = h5py.File(filename, 'w')
+
+        g1 = f.create_group("g1", track_order=True)  # creation order
+        self.fill_attrs(g1)
+        title = 'three'
+        self.assertEqual(g1.attrs[title], 3)
         # overwrite attribute
-        group.attrs['11'] = 42.0
-        self.assertEqual(group.attrs["11"], 42.0)
+        g1.attrs[title] = 42.0
+        self.assertEqual(g1.attrs[title], 42.0)
         # delete attribute
-        self.assertIn('10', group.attrs)
-        del group.attrs['10']
-        self.assertNotIn('10', group.attrs)
+        self.assertIn(title, g1.attrs)
+        del g1.attrs[title]
+        self.assertNotIn(title, g1.attrs)
 
     def test_track_order_not_inherited(self):
         """
@@ -343,13 +362,11 @@ def test_track_order_not_inherited(self):
         """
         filename = self.getFileName("test_test_track_order_not_inherited")
         print(f"filename: {filename}")
-        self.f = h5py.File(filename, 'w', track_order=True)
-        group = self.f.create_group('test')
-
-        for i in range(12):
-            group.attrs[str(i)] = i
+        f = h5py.File(filename, 'w', track_order=True)
+        g1 = f.create_group('test')
+        self.fill_attrs(g1)
 
-        self.assertEqual(list(group.attrs), sorted([str(i) for i in range(12)]))
+        self.assertEqual(list(g1.attrs), sorted(list(self.titles)))
 
 
 if __name__ == '__main__':

test/hl/test_file.py

@@ -350,39 +350,65 @@ def test_close(self):
 
 
 class TestTrackOrder(TestCase):
+    titles = ("one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten")
+
     def populate(self, f):
-        count = 3
+        count = len(self.titles)
+        # create count datasets/groups
         for i in range(count):
+            title = self.titles[i]
             # Mix group and dataset creation.
-            if i % 10 == 0:
-                f.create_group(str(i))
+            if i % 2 == 0:
+                f.create_group(title)
             else:
-                f[str(i)] = [i]
-        return count
+                f[title] = [i]
+        # create count attributes
+        for i in range(count):
+            title = self.titles[i]
+            f.attrs[title] = i
 
     def test_track_order(self):
         filename = self.getFileName("test_track_order_file")
         print(f"filename: {filename}")
         # write file using creation order
         with h5py.File(filename, 'w', track_order=True) as f:
-            count = self.populate(f)
-            self.assertEqual(list(f), [str(i) for i in range(count)])
+            self.populate(f)
+            self.assertEqual(list(f), list(self.titles))
+            self.assertEqual(list(f.attrs), list(self.titles))
+
+        with h5py.File(filename) as f:
+            # domain/file should have been saved with track_order state
+            self.assertEqual(list(f), list(self.titles))
+            self.assertEqual(list(f.attrs), list(self.titles))
+
+    def test_cfg_track_order(self):
+        filename = self.getFileName("test_cfg_track_order_file")
+        print(f"filename: {filename}")
+        # write file using creation order
+        cfg = h5py.get_config()
+        cfg.track_order = True
+        with h5py.File(filename, 'w') as f:
+            self.populate(f)
+            self.assertEqual(list(f), list(self.titles))
+            self.assertEqual(list(f.attrs), list(self.titles))
+        cfg.track_order = False  # reset
 
         with h5py.File(filename) as f:
             # domain/file should have been saved with track_order state
-            self.assertEqual(list(f), [str(i) for i in range(count)])
+            self.assertEqual(list(f), list(self.titles))
+            self.assertEqual(list(f.attrs), list(self.titles))
 
     def test_no_track_order(self):
         filename = self.getFileName("test_no_track_order_file")
         print(f"filename: {filename}")
 
         # create file using alphanumeric order
         with h5py.File(filename, 'w', track_order=False) as f:
-            count = self.populate(f)
-            self.assertEqual(list(f), sorted([str(i) for i in range(count)]))
+            self.populate(f)
+            self.assertEqual(list(f), sorted(self.titles))
 
         with h5py.File(filename) as f:  # name alphanumeric
-            self.assertEqual(list(f), sorted([str(i) for i in range(count)]))
+            self.assertEqual(list(f), sorted(self.titles))
 
 
 if __name__ == '__main__':