update datumaro

Dockerfile

@@ -21,6 +21,7 @@
         libxmlsec1-dev \
         libxmlsec1-openssl \
         libhdf5-dev \
+        cargo \
     && rm -rf /var/lib/apt/lists/*
 
 ARG PIP_VERSION
@@ -93,7 +94,7 @@
 
 ARG http_proxy
 ARG https_proxy
 ARG no_proxy="nuclio,${no_proxy}"
 ARG socks_proxy
 ARG TZ="Etc/UTC"
 
@@ -145,7 +146,7 @@
 
 # Add a non-root user
 ENV USER=${USER}
 ENV HOME /home/${USER}
 RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER}
 
 ARG CLAM_AV="no"

cvat/apps/dataset_manager/bindings.py

@@ -1639,7 +1639,7 @@ def map_label(name, parent=''): return label_cat.find(name, parent)[0]
         return self.convert_annotations(cvat_frame_anno, label_attrs, map_label)
 
 
-class CvatTaskOrJobDataExtractor(dm.SourceExtractor, CVATDataExtractorMixin):
+class CvatTaskOrJobDataExtractor(dm.SubsetBase, CVATDataExtractorMixin):
     def __init__(
         self,
         instance_data: CommonData,
@@ -1650,7 +1650,7 @@ def __init__(
         **kwargs
     ):
         instance_meta = instance_data.meta[instance_data.META_FIELD]
-        dm.SourceExtractor.__init__(
+        dm.SubsetBase.__init__(
             self,
             media_type=dm.Image if dimension == DimensionType.DIM_2D else dm.PointCloud,
             subset=instance_meta['subset'],
@@ -1748,7 +1748,7 @@ def map_label(name, parent=''): return label_cat.find(name, parent)[0]
         return self.convert_annotations(cvat_frame_anno,
             label_attrs, map_label, self._format_type, self._dimension)
 
-class CVATProjectDataExtractor(dm.Extractor, CVATDataExtractorMixin):
+class CVATProjectDataExtractor(dm.DatasetBase, CVATDataExtractorMixin):
     def __init__(
         self,
         project_data: ProjectData,
@@ -1758,7 +1758,7 @@ def __init__(
         dimension: DimensionType = DimensionType.DIM_2D,
         **kwargs
     ):
-        dm.Extractor.__init__(
+        dm.DatasetBase.__init__(
             self, media_type=dm.Image if dimension == DimensionType.DIM_2D else dm.PointCloud
         )
         CVATDataExtractorMixin.__init__(self, **kwargs)

cvat/apps/dataset_manager/formats/cvat.py

@@ -22,7 +22,8 @@
     Skeleton,
 )
 from datumaro.components.dataset import Dataset, DatasetItem
-from datumaro.components.extractor import DEFAULT_SUBSET_NAME, Extractor, Importer
+from datumaro.components.dataset_base import DEFAULT_SUBSET_NAME, DatasetBase
+from datumaro.components.importer import Importer
 from datumaro.plugins.data_formats.cvat.base import CvatImporter as _CvatImporter
 from datumaro.util.image import Image
 from defusedxml import ElementTree
@@ -50,7 +51,7 @@ class CvatPath:
 
     BUILTIN_ATTRS = {'occluded', 'outside', 'keyframe', 'track_id'}
 
-class CvatExtractor(Extractor):
+class CvatExtractor(DatasetBase):
     _SUPPORTED_SHAPES = ('box', 'polygon', 'polyline', 'points', 'skeleton')
 
     def __init__(self, path, subsets=None):

cvat/apps/dataset_manager/formats/icdar.py

@@ -7,7 +7,7 @@
 
 from datumaro.components.annotation import AnnotationType, Caption, Label, LabelCategories
 from datumaro.components.dataset import Dataset
-from datumaro.components.extractor import ItemTransform
+from datumaro.components.transformer import ItemTransform
 
 from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, import_dm_annotations
 from cvat.apps.dataset_manager.util import make_zip_archive

cvat/apps/dataset_manager/formats/market1501.py

@@ -7,7 +7,7 @@
 
 from datumaro.components.annotation import AnnotationType, Label, LabelCategories
 from datumaro.components.dataset import Dataset
-from datumaro.components.extractor import ItemTransform
+from datumaro.components.transformer import ItemTransform
 
 from cvat.apps.dataset_manager.bindings import (
     GetCVATDataExtractor,

cvat/apps/dataset_manager/formats/mots.py

@@ -5,7 +5,7 @@
 
 from datumaro.components.annotation import AnnotationType
 from datumaro.components.dataset import Dataset
-from datumaro.components.extractor import ItemTransform
+from datumaro.components.transformer import ItemTransform
 from pyunpack import Archive
 
 from cvat.apps.dataset_manager.bindings import (

cvat/apps/dataset_manager/formats/velodynepoint.py

@@ -6,7 +6,7 @@
 import zipfile
 
 from datumaro.components.dataset import Dataset
-from datumaro.components.extractor import ItemTransform
+from datumaro.components.transformer import ItemTransform
 
 from cvat.apps.dataset_manager.bindings import (
     GetCVATDataExtractor,

cvat/apps/dataset_manager/formats/yolo.py

@@ -7,7 +7,7 @@
 from typing import Callable, Optional
 
 from datumaro.components.annotation import AnnotationType
-from datumaro.components.extractor import DatasetItem
+from datumaro.components.dataset_base import DatasetItem
 from datumaro.components.project import Dataset
 from pyunpack import Archive
 

cvat/apps/dataset_manager/tests/test_formats.py

@@ -678,10 +678,7 @@ def test_dataset_root(self):
                 task = self._generate_task(images)
                 task_data = TaskData(AnnotationIR('2d'),
                     Task.objects.get(pk=task["id"]))
-                dataset = [
-                    datumaro.components.extractor.DatasetItem(
-                        id=osp.splitext(p)[0])
-                    for p in dataset_paths]
+                dataset = [DatasetItem(id=osp.splitext(p)[0]) for p in dataset_paths]
 
                 root = find_dataset_root(dataset, task_data)
                 self.assertEqual(expected, root)

cvat/requirements/base.in

@@ -12,7 +12,7 @@ azure-storage-blob==12.13.0
 boto3==1.17.61
 clickhouse-connect==0.6.8
 coreapi==2.3.3
-datumaro @ git+https://github.com/cvat-ai/datumaro.git@08e77b216080555a57e12c01625be8c8201e3131
+datumaro @ git+https://github.com/cvat-ai/datumaro.git@ebcc2d4254e5b5c19b51ad6062dc882366871703
 dj-pagination==2.5.0
 # Despite direct indication allauth in requirements we should keep 'with_social' for dj-rest-auth
 # to avoid possible further versions conflicts (we use registration functionality)

cvat/requirements/base.txt

@@ -1,4 +1,4 @@
-# SHA1:3e6349d9e5e095c5a1f196eca66b3e5ba8672458
+# SHA1:062a211315044f19a89f15ed000c974a1d2eff14
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
@@ -39,7 +39,9 @@ cffi==1.17.1
 charset-normalizer==3.4.1
     # via requests
 click==8.1.8
-    # via rq
+    # via
+    #   nltk
+    #   rq
 clickhouse-connect==0.6.8
     # via -r cvat/requirements/base.in
 contourpy==1.2.1
@@ -53,10 +55,11 @@ crontab==1.0.1
 cryptography==44.0.0
     # via
     #   azure-storage-blob
+    #   datumaro
     #   pyjwt
 cycler==0.12.1
     # via matplotlib
-datumaro @ git+https://github.com/cvat-ai/datumaro.git@08e77b216080555a57e12c01625be8c8201e3131
+datumaro @ git+https://github.com/cvat-ai/datumaro.git@ebcc2d4254e5b5c19b51ad6062dc882366871703
     # via -r cvat/requirements/base.in
 defusedxml==0.7.1
     # via
@@ -145,6 +148,8 @@ h5py==3.12.1
     # via datumaro
 idna==3.10
     # via requests
+imagesize==1.4.1
+    # via datumaro
 importlib-metadata==8.5.0
     # via clickhouse-connect
 importlib-resources==6.5.2
@@ -165,6 +170,14 @@ jmespath==0.10.0
     # via
     #   boto3
     #   botocore
+joblib==1.4.2
+    # via
+    #   nltk
+    #   scikit-learn
+json-stream==2.3.2
+    # via datumaro
+json-stream-rs-tokenizer==0.4.27
+    # via json-stream
 jsonschema==4.17.3
     # via drf-spectacular
 kiwisolver==1.4.7
@@ -193,11 +206,13 @@ networkx==3.2.1
     # via datumaro
 nibabel==5.3.2
     # via datumaro
+nltk==3.9.1
+    # via datumaro
 oauthlib==3.2.2
     # via requests-oauthlib
 orderedmultidict==1.0.1
     # via furl
-orjson==3.10.13
+orjson==3.10.12
     # via datumaro
 packaging==24.2
     # via
@@ -211,12 +226,15 @@ patool==1.12
     # via -r cvat/requirements/base.in
 pdf2image==1.14.0
     # via -r cvat/requirements/base.in
+portalocker==3.1.1
+    # via datumaro
 pottery==3.0.0
     # via -r cvat/requirements/base.in
 proto-plus==1.25.0
     # via google-api-core
-protobuf==5.29.2
+protobuf==5.29.3
     # via
+    #   datumaro
     #   google-api-core
     #   googleapis-common-protos
     #   proto-plus
@@ -225,6 +243,8 @@ psutil==5.9.4
     # via -r cvat/requirements/base.in
 psycopg2-binary==2.9.5
     # via -r cvat/requirements/base.in
+pyarrow==18.1.0
+    # via datumaro
 pyasn1==0.6.1
     # via
     #   pyasn1-modules
@@ -238,6 +258,8 @@ pycocotools==2.0.8
     # via datumaro
 pycparser==2.22
     # via cffi
+pyemd==1.0.0
+    # via datumaro
 pyjwt[crypto]==2.10.1
     # via django-allauth
 pylogbeat==2.0.1
@@ -281,6 +303,8 @@ redis==4.6.0
     #   django-rq
     #   pottery
     #   rq
+regex==2024.11.6
+    # via nltk
 requests==2.32.3
     # via
     #   -r cvat/requirements/base.in
@@ -316,10 +340,16 @@ rules==3.5
     # via -r cvat/requirements/base.in
 s3transfer==0.4.2
     # via boto3
-scipy==1.13.1
+scikit-learn==1.6.1
     # via datumaro
+scipy==1.13.1
+    # via
+    #   datumaro
+    #   scikit-learn
 shapely==1.7.1
-    # via -r cvat/requirements/base.in
+    # via
+    #   -r cvat/requirements/base.in
+    #   datumaro
 six==1.17.0
     # via
     #   azure-core
@@ -328,8 +358,12 @@ six==1.17.0
     #   python-dateutil
 sqlparse==0.5.3
     # via django
+tabulate==0.9.0
+    # via datumaro
 tensorboardx==2.6.2.2
     # via datumaro
+threadpoolctl==3.5.0
+    # via scikit-learn
 typing-extensions==4.12.2
     # via
     #   asgiref
@@ -349,7 +383,7 @@ urllib3==1.26.20
     #   botocore
     #   clickhouse-connect
     #   requests
-wrapt==1.17.0
+wrapt==1.17.1
     # via deprecated
 xmlsec==1.3.14
     # via

cvat/requirements/production.txt

@@ -24,7 +24,7 @@ uvicorn[standard]==0.22.0
     # via -r cvat/requirements/production.in
 uvloop==0.21.0
     # via uvicorn
-watchfiles==1.0.3
+watchfiles==1.0.4
     # via uvicorn
 websockets==14.1
     # via uvicorn