From 76de839ab25ee8c8fc74da499065214397786cd7 Mon Sep 17 00:00:00 2001 From: Roman Donchenko Date: Mon, 6 Jan 2025 15:55:26 +0200 Subject: [PATCH 1/3] Apply isort to almost all Python sources (#8866) isort has a `--resolve-all-configs` options that allows you to use different configs for different subdirectories. With this option, we can reformat/check the whole codebase with one command. So let's just do that. It's a little janky, because isort's first party module detection doesn't seem to work properly when you run it from the project root and not the appropriate subdirectory. So I had to patch that up with explicit `known_first_party` settings. I also didn't feel like doing that for all of the serverless functions, so I just added them to the ignore list for now. --- .github/workflows/isort.yml | 30 +-- cvat-sdk/pyproject.toml | 1 + cvat/apps/dataset_manager/annotation.py | 10 +- cvat/apps/dataset_manager/bindings.py | 28 ++- cvat/apps/dataset_manager/formats/camvid.py | 5 +- .../dataset_manager/formats/cityscapes.py | 9 +- cvat/apps/dataset_manager/formats/coco.py | 8 +- cvat/apps/dataset_manager/formats/cvat.py | 23 ++- cvat/apps/dataset_manager/formats/datumaro.py | 6 +- cvat/apps/dataset_manager/formats/icdar.py | 8 +- cvat/apps/dataset_manager/formats/imagenet.py | 3 +- cvat/apps/dataset_manager/formats/kitti.py | 9 +- cvat/apps/dataset_manager/formats/labelme.py | 7 +- cvat/apps/dataset_manager/formats/lfw.py | 7 +- .../dataset_manager/formats/market1501.py | 11 +- cvat/apps/dataset_manager/formats/mask.py | 10 +- cvat/apps/dataset_manager/formats/mots.py | 10 +- .../dataset_manager/formats/openimages.py | 11 +- .../dataset_manager/formats/pascal_voc.py | 6 +- .../dataset_manager/formats/pointcloud.py | 7 +- .../formats/transformations.py | 6 +- cvat/apps/dataset_manager/formats/utils.py | 5 +- .../dataset_manager/formats/velodynepoint.py | 12 +- cvat/apps/dataset_manager/formats/vggface2.py | 8 +- .../apps/dataset_manager/formats/widerface.py | 7 +- cvat/apps/dataset_manager/formats/yolo.py | 8 +- cvat/apps/dataset_manager/project.py | 12 +- cvat/apps/dataset_manager/task.py | 27 ++- .../dataset_manager/tests/test_formats.py | 17 +- .../tests/test_rest_api_formats.py | 15 +- cvat/apps/dataset_manager/views.py | 16 +- .../dataset_repo/migrations/0001_initial.py | 2 +- .../dataset_repo/migrations/0004_rename.py | 1 + cvat/apps/engine/__init__.py | 2 +- cvat/apps/engine/admin.py | 17 +- cvat/apps/engine/apps.py | 1 + cvat/apps/engine/backup.py | 58 ++++-- cvat/apps/engine/cloud_provider.py | 10 +- cvat/apps/engine/filters.py | 18 +- cvat/apps/engine/frame_provider.py | 10 +- cvat/apps/engine/handlers.py | 2 + cvat/apps/engine/location.py | 5 +- cvat/apps/engine/log.py | 7 +- .../management/commands/syncperiodicjobs.py | 4 +- cvat/apps/engine/media_extractors.py | 19 +- cvat/apps/engine/middleware.py | 1 + .../engine/migrations/0001_release_v0_1_0.py | 2 +- ...dpolygonattributeval_labeledpolyline_la.py | 2 +- .../migrations/0008_auto_20180917_1424.py | 2 +- .../0011_add_task_source_and_safecharfield.py | 3 +- .../0013_auth_no_default_permissions.py | 2 +- .../migrations/0015_db_redesign_20190217.py | 6 +- .../0016_attribute_spec_20190217.py | 9 +- .../migrations/0017_db_redesign_20190221.py | 6 +- cvat/apps/engine/migrations/0018_jobcommit.py | 2 +- .../migrations/0020_remove_task_flipped.py | 11 +- .../migrations/0022_auto_20191004_0817.py | 5 +- .../migrations/0023_auto_20200113_1323.py | 3 +- .../migrations/0024_auto_20191023_1025.py | 26 ++- .../apps/engine/migrations/0028_labelcolor.py | 2 + .../migrations/0029_data_storage_method.py | 9 +- .../migrations/0033_projects_adjastment.py | 2 +- .../migrations/0034_auto_20201125_1426.py | 6 +- .../engine/migrations/0035_data_storage.py | 3 +- .../migrations/0036_auto_20201216_0943.py | 5 +- cvat/apps/engine/migrations/0038_manifest.py | 3 +- .../engine/migrations/0039_auto_training.py | 2 +- .../engine/migrations/0040_cloud_storage.py | 5 +- .../migrations/0042_auto_20210830_1056.py | 2 +- .../migrations/0046_data_sorting_method.py | 3 +- .../migrations/0047_auto_20211110_1938.py | 5 +- .../migrations/0048_auto_20211112_1918.py | 2 +- .../migrations/0053_data_deleted_frames.py | 3 +- .../migrations/0054_auto_20220610_1829.py | 5 +- .../migrations/0055_jobs_directories.py | 3 +- .../engine/migrations/0056_jobs_previews.py | 4 +- .../migrations/0057_auto_20220726_0926.py | 5 +- .../migrations/0058_auto_20220809_1236.py | 2 +- .../migrations/0060_alter_label_parent.py | 2 +- .../engine/migrations/0062_delete_previews.py | 4 +- .../0064_delete_or_rename_wrong_labels.py | 2 + .../0070_add_job_type_created_date.py | 5 +- .../migrations/0071_annotationguide_asset.py | 5 +- .../0072_alter_issue_updated_date.py | 1 + ...es_that_refer_to_deleted_cloud_storages.py | 1 + .../migrations/0077_auto_20231121_1952.py | 2 +- ...labeledimageattributeval_image_and_more.py | 2 +- .../0080_alter_trackedshape_track.py | 2 +- .../0082_alter_labeledimage_job_and_more.py | 2 +- .../0085_segment_chunks_updated_date.py | 1 + cvat/apps/engine/mime_types.py | 3 +- cvat/apps/engine/mixins.py | 14 +- cvat/apps/engine/pagination.py | 2 + cvat/apps/engine/parsers.py | 1 + cvat/apps/engine/permissions.py | 13 +- cvat/apps/engine/renderers.py | 1 + cvat/apps/engine/rq_job_handler.py | 9 +- cvat/apps/engine/serializers.py | 48 +++-- cvat/apps/engine/signals.py | 3 +- cvat/apps/engine/task.py | 40 ++-- cvat/apps/engine/tests/test_lazy_list.py | 4 +- cvat/apps/engine/tests/test_rest_api.py | 48 +++-- cvat/apps/engine/tests/test_rest_api_3D.py | 9 +- cvat/apps/engine/tests/utils.py | 12 +- cvat/apps/engine/urls.py | 11 +- cvat/apps/engine/utils.py | 47 +++-- cvat/apps/engine/view_utils.py | 2 +- cvat/apps/engine/views.py | 182 ++++++++++++------ cvat/apps/events/apps.py | 4 +- cvat/apps/events/cache.py | 2 +- cvat/apps/events/event.py | 3 +- cvat/apps/events/export.py | 14 +- cvat/apps/events/handlers.py | 42 ++-- cvat/apps/events/permissions.py | 2 +- cvat/apps/events/tests/test_events.py | 5 +- cvat/apps/events/utils.py | 5 +- cvat/apps/events/views.py | 3 +- cvat/apps/health/apps.py | 2 +- cvat/apps/health/backends.py | 5 +- .../health/management/commands/workerprobe.py | 5 +- cvat/apps/iam/adapters.py | 4 +- cvat/apps/iam/admin.py | 2 +- cvat/apps/iam/apps.py | 1 + cvat/apps/iam/authentication.py | 8 +- cvat/apps/iam/filters.py | 4 +- cvat/apps/iam/forms.py | 7 +- cvat/apps/iam/middleware.py | 4 +- .../migrations/0001_remove_business_group.py | 1 - cvat/apps/iam/rules/tests/generate_tests.py | 2 +- cvat/apps/iam/schema.py | 1 - cvat/apps/iam/serializers.py | 18 +- cvat/apps/iam/signals.py | 5 +- cvat/apps/iam/tests/test_rest_api.py | 10 +- cvat/apps/iam/urls.py | 20 +- cvat/apps/iam/utils.py | 4 +- cvat/apps/iam/views.py | 35 ++-- cvat/apps/lambda_manager/models.py | 1 + cvat/apps/lambda_manager/permissions.py | 1 + cvat/apps/lambda_manager/serializers.py | 1 + cvat/apps/lambda_manager/tests/test_lambda.py | 10 +- cvat/apps/lambda_manager/views.py | 33 ++-- cvat/apps/log_viewer/views.py | 2 +- cvat/apps/organizations/admin.py | 4 +- cvat/apps/organizations/apps.py | 1 + .../organizations/migrations/0001_initial.py | 2 +- cvat/apps/organizations/models.py | 13 +- cvat/apps/organizations/permissions.py | 1 + cvat/apps/organizations/serializers.py | 10 +- cvat/apps/organizations/throttle.py | 1 + cvat/apps/organizations/urls.py | 1 + cvat/apps/organizations/views.py | 33 ++-- cvat/apps/webhooks/apps.py | 4 +- cvat/apps/webhooks/migrations/0001_initial.py | 5 +- cvat/apps/webhooks/permissions.py | 2 +- cvat/apps/webhooks/serializers.py | 9 +- cvat/apps/webhooks/signals.py | 16 +- cvat/apps/webhooks/urls.py | 1 + cvat/apps/webhooks/views.py | 18 +- cvat/settings/base.py | 5 +- cvat/settings/email_settings.py | 1 - cvat/settings/testing.py | 7 +- cvat/urls.py | 2 +- cvat/utils/http.py | 3 +- dev/update_version.py | 1 - pyproject.toml | 8 + rqscheduler.py | 4 +- tests/python/pyproject.toml | 1 + utils/dataset_manifest/__init__.py | 2 +- utils/dataset_manifest/core.py | 17 +- utils/dataset_manifest/create.py | 7 +- utils/dataset_manifest/types.py | 1 + utils/dataset_manifest/utils.py | 10 +- utils/dicom_converter/script.py | 5 +- 173 files changed, 951 insertions(+), 637 deletions(-) diff --git a/.github/workflows/isort.yml b/.github/workflows/isort.yml index bf90604cbb2f..620dc6c85d79 100644 --- a/.github/workflows/isort.yml +++ b/.github/workflows/isort.yml @@ -5,35 +5,11 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - id: files - uses: tj-actions/changed-files@v41.0.0 - with: - files: | - cvat-sdk/**/*.py - cvat-cli/**/*.py - tests/python/**/*.py - cvat/apps/quality_control/**/*.py - cvat/apps/analytics_report/**/*.py - dir_names: true - name: Run checks run: | - # If different modules use different isort configs, - # we need to run isort for each python component group separately. - # Otherwise, they all will use the same config. + pipx install $(grep "^isort" ./dev/requirements.txt) - UPDATED_DIRS="${{steps.files.outputs.all_changed_files}}" + echo "isort version: $(isort --version-number)" - if [[ ! -z $UPDATED_DIRS ]]; then - pipx install $(grep "^isort" ./dev/requirements.txt) - - echo "isort version: $(isort --version-number)" - echo "The dirs will be checked: $UPDATED_DIRS" - EXIT_CODE=0 - for DIR in $UPDATED_DIRS; do - isort --check $DIR || EXIT_CODE=$(($? | $EXIT_CODE)) || true - done - exit $EXIT_CODE - else - echo "No files with the \"py\" extension found" - fi + isort --check --diff --resolve-all-configs . diff --git a/cvat-sdk/pyproject.toml b/cvat-sdk/pyproject.toml index ce8cba3ffba6..8d3fb7787504 100644 --- a/cvat-sdk/pyproject.toml +++ b/cvat-sdk/pyproject.toml @@ -7,3 +7,4 @@ profile = "black" forced_separate = ["tests"] line_length = 100 skip_gitignore = true # align tool behavior with Black +known_first_party = ["cvat_sdk"] diff --git a/cvat/apps/dataset_manager/annotation.py b/cvat/apps/dataset_manager/annotation.py index 4ea10ba9619d..943e53d003e3 100644 --- a/cvat/apps/dataset_manager/annotation.py +++ b/cvat/apps/dataset_manager/annotation.py @@ -3,19 +3,19 @@ # # SPDX-License-Identifier: MIT -from copy import copy, deepcopy - import math from collections.abc import Container, Sequence +from copy import copy, deepcopy +from itertools import chain from typing import Optional + import numpy as np -from itertools import chain from scipy.optimize import linear_sum_assignment from shapely import geometry -from cvat.apps.engine.models import ShapeType, DimensionType -from cvat.apps.engine.serializers import LabeledDataSerializer from cvat.apps.dataset_manager.util import faster_deepcopy +from cvat.apps.engine.models import DimensionType, ShapeType +from cvat.apps.engine.serializers import LabeledDataSerializer class AnnotationIR: diff --git a/cvat/apps/dataset_manager/bindings.py b/cvat/apps/dataset_manager/bindings.py index 8b759f7b6316..a4688d80ba5a 100644 --- a/cvat/apps/dataset_manager/bindings.py +++ b/cvat/apps/dataset_manager/bindings.py @@ -16,29 +16,39 @@ from types import SimpleNamespace from typing import Any, Callable, Literal, NamedTuple, Optional, Union -from attrs.converters import to_bool import datumaro as dm import defusedxml.ElementTree as ET import rq from attr import attrib, attrs +from attrs.converters import to_bool from datumaro.components.format_detection import RejectionReason +from django.conf import settings from django.db.models import Prefetch, QuerySet from django.utils import timezone -from django.conf import settings from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.dataset_manager.util import add_prefetch_fields from cvat.apps.engine import models -from cvat.apps.engine.frame_provider import TaskFrameProvider, FrameQuality, FrameOutputType -from cvat.apps.engine.models import (AttributeSpec, AttributeType, DimensionType, Job, - JobType, Label, LabelType, Project, SegmentType, ShapeType, - Task) -from cvat.apps.engine.rq_job_handler import RQJobMetaField +from cvat.apps.engine.frame_provider import FrameOutputType, FrameQuality, TaskFrameProvider from cvat.apps.engine.lazy_list import LazyList +from cvat.apps.engine.models import ( + AttributeSpec, + AttributeType, + DimensionType, + Job, + JobType, + Label, + LabelType, + Project, + SegmentType, + ShapeType, + Task, +) +from cvat.apps.engine.rq_job_handler import RQJobMetaField -from .annotation import AnnotationIR, AnnotationManager, TrackManager -from .formats.transformations import MaskConverter, EllipsesToMasks from ..engine.log import ServerLogManager +from .annotation import AnnotationIR, AnnotationManager, TrackManager +from .formats.transformations import EllipsesToMasks, MaskConverter slogger = ServerLogManager(__name__) diff --git a/cvat/apps/dataset_manager/formats/camvid.py b/cvat/apps/dataset_manager/formats/camvid.py index 75cea9e98bd4..e995c5f1075d 100644 --- a/cvat/apps/dataset_manager/formats/camvid.py +++ b/cvat/apps/dataset_manager/formats/camvid.py @@ -6,12 +6,11 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .utils import make_colormap diff --git a/cvat/apps/dataset_manager/formats/cityscapes.py b/cvat/apps/dataset_manager/formats/cityscapes.py index ea39578ea3f3..6de867e551c6 100644 --- a/cvat/apps/dataset_manager/formats/cityscapes.py +++ b/cvat/apps/dataset_manager/formats/cityscapes.py @@ -9,12 +9,15 @@ from datumaro.plugins.cityscapes_format import write_label_map from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .utils import make_colormap diff --git a/cvat/apps/dataset_manager/formats/coco.py b/cvat/apps/dataset_manager/formats/coco.py index 1d1a8ce4d0d5..bddb33b08ba9 100644 --- a/cvat/apps/dataset_manager/formats/coco.py +++ b/cvat/apps/dataset_manager/formats/coco.py @@ -5,17 +5,21 @@ import zipfile -from datumaro.components.dataset import Dataset from datumaro.components.annotation import AnnotationType +from datumaro.components.dataset import Dataset from datumaro.plugins.coco_format.importer import CocoImporter from cvat.apps.dataset_manager.bindings import ( - GetCVATDataExtractor, NoMediaInAnnotationFileError, import_dm_annotations, detect_dataset + GetCVATDataExtractor, + NoMediaInAnnotationFileError, + detect_dataset, + import_dm_annotations, ) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer + @exporter(name='COCO', ext='ZIP', version='1.0') def _export(dst_file, temp_dir, instance_data, save_images=False): with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor: diff --git a/cvat/apps/dataset_manager/formats/cvat.py b/cvat/apps/dataset_manager/formats/cvat.py index fa46b58813bf..58a8076f01cc 100644 --- a/cvat/apps/dataset_manager/formats/cvat.py +++ b/cvat/apps/dataset_manager/formats/cvat.py @@ -11,29 +11,34 @@ from io import BufferedWriter from typing import Callable, Union -from datumaro.components.annotation import (AnnotationType, Bbox, Label, - LabelCategories, Points, Polygon, - PolyLine, Skeleton) +from datumaro.components.annotation import ( + AnnotationType, + Bbox, + Label, + LabelCategories, + Points, + Polygon, + PolyLine, + Skeleton, +) from datumaro.components.dataset import Dataset, DatasetItem -from datumaro.components.extractor import (DEFAULT_SUBSET_NAME, Extractor, - Importer) +from datumaro.components.extractor import DEFAULT_SUBSET_NAME, Extractor, Importer from datumaro.plugins.cvat_format.extractor import CvatImporter as _CvatImporter - from datumaro.util.image import Image from defusedxml import ElementTree from cvat.apps.dataset_manager.bindings import ( + JobData, NoMediaInAnnotationFileError, ProjectData, TaskData, - JobData, detect_dataset, get_defaulted_subset, import_dm_annotations, - match_dm_item + match_dm_item, ) from cvat.apps.dataset_manager.util import make_zip_archive -from cvat.apps.engine.frame_provider import FrameQuality, FrameOutputType, make_frame_provider +from cvat.apps.engine.frame_provider import FrameOutputType, FrameQuality, make_frame_provider from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/formats/datumaro.py b/cvat/apps/dataset_manager/formats/datumaro.py index 4fc1d246dd47..81f86cb32065 100644 --- a/cvat/apps/dataset_manager/formats/datumaro.py +++ b/cvat/apps/dataset_manager/formats/datumaro.py @@ -4,10 +4,14 @@ # SPDX-License-Identifier: MIT import zipfile + from datumaro.components.dataset import Dataset from cvat.apps.dataset_manager.bindings import ( - GetCVATDataExtractor, import_dm_annotations, NoMediaInAnnotationFileError, detect_dataset + GetCVATDataExtractor, + NoMediaInAnnotationFileError, + detect_dataset, + import_dm_annotations, ) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType diff --git a/cvat/apps/dataset_manager/formats/icdar.py b/cvat/apps/dataset_manager/formats/icdar.py index 5d031eef82b0..c72f9708fe11 100644 --- a/cvat/apps/dataset_manager/formats/icdar.py +++ b/cvat/apps/dataset_manager/formats/icdar.py @@ -5,17 +5,15 @@ import zipfile -from datumaro.components.annotation import (AnnotationType, Caption, Label, - LabelCategories) +from datumaro.components.annotation import AnnotationType, Caption, Label, LabelCategories from datumaro.components.dataset import Dataset from datumaro.components.extractor import ItemTransform -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons class AddLabelToAnns(ItemTransform): diff --git a/cvat/apps/dataset_manager/formats/imagenet.py b/cvat/apps/dataset_manager/formats/imagenet.py index fd5e9a99a176..273f47616bc1 100644 --- a/cvat/apps/dataset_manager/formats/imagenet.py +++ b/cvat/apps/dataset_manager/formats/imagenet.py @@ -9,8 +9,7 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, \ - import_dm_annotations +from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, import_dm_annotations from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/formats/kitti.py b/cvat/apps/dataset_manager/formats/kitti.py index 01e1cd3fc4bc..0bdfdc4ab6e6 100644 --- a/cvat/apps/dataset_manager/formats/kitti.py +++ b/cvat/apps/dataset_manager/formats/kitti.py @@ -7,14 +7,17 @@ from datumaro.components.dataset import Dataset from datumaro.plugins.kitti_format.format import KittiPath, write_label_map - from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .utils import make_colormap diff --git a/cvat/apps/dataset_manager/formats/labelme.py b/cvat/apps/dataset_manager/formats/labelme.py index be9679f268e8..179fb320f322 100644 --- a/cvat/apps/dataset_manager/formats/labelme.py +++ b/cvat/apps/dataset_manager/formats/labelme.py @@ -6,8 +6,11 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.formats.transformations import MaskToPolygonTransformation from cvat.apps.dataset_manager.util import make_zip_archive diff --git a/cvat/apps/dataset_manager/formats/lfw.py b/cvat/apps/dataset_manager/formats/lfw.py index 0af356332bb5..407240c5e0a3 100644 --- a/cvat/apps/dataset_manager/formats/lfw.py +++ b/cvat/apps/dataset_manager/formats/lfw.py @@ -6,8 +6,11 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/formats/market1501.py b/cvat/apps/dataset_manager/formats/market1501.py index 6be8b2fcf75f..e9d46a095bc8 100644 --- a/cvat/apps/dataset_manager/formats/market1501.py +++ b/cvat/apps/dataset_manager/formats/market1501.py @@ -5,17 +5,20 @@ import zipfile -from datumaro.components.annotation import (AnnotationType, Label, - LabelCategories) +from datumaro.components.annotation import AnnotationType, Label, LabelCategories from datumaro.components.dataset import Dataset from datumaro.components.extractor import ItemTransform -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer + class AttrToLabelAttr(ItemTransform): def __init__(self, extractor, label): super().__init__(extractor) diff --git a/cvat/apps/dataset_manager/formats/mask.py b/cvat/apps/dataset_manager/formats/mask.py index f003f68383e7..eab4238f4242 100644 --- a/cvat/apps/dataset_manager/formats/mask.py +++ b/cvat/apps/dataset_manager/formats/mask.py @@ -6,14 +6,18 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .utils import make_colormap + @exporter(name='Segmentation mask', ext='ZIP', version='1.1') def _export(dst_file, temp_dir, instance_data, save_images=False): with GetCVATDataExtractor(instance_data, include_images=save_images) as extractor: diff --git a/cvat/apps/dataset_manager/formats/mots.py b/cvat/apps/dataset_manager/formats/mots.py index 9ed156e6cd4e..736ccb1ce0f8 100644 --- a/cvat/apps/dataset_manager/formats/mots.py +++ b/cvat/apps/dataset_manager/formats/mots.py @@ -8,12 +8,16 @@ from datumaro.components.extractor import ItemTransform from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - find_dataset_root, match_dm_item) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + find_dataset_root, + match_dm_item, +) from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons class KeepTracks(ItemTransform): diff --git a/cvat/apps/dataset_manager/formats/openimages.py b/cvat/apps/dataset_manager/formats/openimages.py index 51fcee29a2fb..c383a64e188a 100644 --- a/cvat/apps/dataset_manager/formats/openimages.py +++ b/cvat/apps/dataset_manager/formats/openimages.py @@ -11,12 +11,17 @@ from datumaro.util.image import DEFAULT_IMAGE_META_FILE_NAME from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - find_dataset_root, import_dm_annotations, match_dm_item) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + find_dataset_root, + import_dm_annotations, + match_dm_item, +) from cvat.apps.dataset_manager.util import make_zip_archive -from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons from .registry import dm_env, exporter, importer +from .transformations import MaskToPolygonTransformation, RotatedBoxesToPolygons def find_item_ids(path): diff --git a/cvat/apps/dataset_manager/formats/pascal_voc.py b/cvat/apps/dataset_manager/formats/pascal_voc.py index a0d84b745d73..3b55928e1f90 100644 --- a/cvat/apps/dataset_manager/formats/pascal_voc.py +++ b/cvat/apps/dataset_manager/formats/pascal_voc.py @@ -11,7 +11,11 @@ from datumaro.components.dataset import Dataset from pyunpack import Archive -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.formats.transformations import MaskToPolygonTransformation from cvat.apps.dataset_manager.util import make_zip_archive diff --git a/cvat/apps/dataset_manager/formats/pointcloud.py b/cvat/apps/dataset_manager/formats/pointcloud.py index 6ddfbb495427..8743c6eb8f3c 100644 --- a/cvat/apps/dataset_manager/formats/pointcloud.py +++ b/cvat/apps/dataset_manager/formats/pointcloud.py @@ -7,8 +7,11 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import (GetCVATDataExtractor, detect_dataset, - import_dm_annotations) +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType diff --git a/cvat/apps/dataset_manager/formats/transformations.py b/cvat/apps/dataset_manager/formats/transformations.py index 99d754252378..a8d361a00168 100644 --- a/cvat/apps/dataset_manager/formats/transformations.py +++ b/cvat/apps/dataset_manager/formats/transformations.py @@ -4,12 +4,12 @@ # SPDX-License-Identifier: MIT import math -import cv2 -import numpy as np from itertools import chain -from pycocotools import mask as mask_utils +import cv2 import datumaro as dm +import numpy as np +from pycocotools import mask as mask_utils class RotatedBoxesToPolygons(dm.ItemTransform): diff --git a/cvat/apps/dataset_manager/formats/utils.py b/cvat/apps/dataset_manager/formats/utils.py index 7811fbbfc902..f565c0aed687 100644 --- a/cvat/apps/dataset_manager/formats/utils.py +++ b/cvat/apps/dataset_manager/formats/utils.py @@ -2,13 +2,14 @@ # # SPDX-License-Identifier: MIT -import os.path as osp -from hashlib import blake2s import itertools import operator +import os.path as osp +from hashlib import blake2s from datumaro.util.os_util import make_file_name + def get_color_from_index(index): def get_bit(number, index): return (number >> index) & 1 diff --git a/cvat/apps/dataset_manager/formats/velodynepoint.py b/cvat/apps/dataset_manager/formats/velodynepoint.py index 9912d0b1d67b..d6051bf6fce8 100644 --- a/cvat/apps/dataset_manager/formats/velodynepoint.py +++ b/cvat/apps/dataset_manager/formats/velodynepoint.py @@ -8,14 +8,16 @@ from datumaro.components.dataset import Dataset from datumaro.components.extractor import ItemTransform -from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, detect_dataset, \ - import_dm_annotations -from .registry import dm_env - +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import DimensionType -from .registry import exporter, importer +from .registry import dm_env, exporter, importer + class RemoveTrackingInformation(ItemTransform): def transform_item(self, item): diff --git a/cvat/apps/dataset_manager/formats/vggface2.py b/cvat/apps/dataset_manager/formats/vggface2.py index 642171f0f8d9..aa172f947db3 100644 --- a/cvat/apps/dataset_manager/formats/vggface2.py +++ b/cvat/apps/dataset_manager/formats/vggface2.py @@ -7,8 +7,12 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, TaskData, detect_dataset, \ - import_dm_annotations +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + TaskData, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/formats/widerface.py b/cvat/apps/dataset_manager/formats/widerface.py index 12a9bf0d21e5..99480bf1f8f5 100644 --- a/cvat/apps/dataset_manager/formats/widerface.py +++ b/cvat/apps/dataset_manager/formats/widerface.py @@ -7,8 +7,11 @@ from datumaro.components.dataset import Dataset -from cvat.apps.dataset_manager.bindings import GetCVATDataExtractor, detect_dataset, \ - import_dm_annotations +from cvat.apps.dataset_manager.bindings import ( + GetCVATDataExtractor, + detect_dataset, + import_dm_annotations, +) from cvat.apps.dataset_manager.util import make_zip_archive from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/formats/yolo.py b/cvat/apps/dataset_manager/formats/yolo.py index 1a138557c862..fac91eae4d35 100644 --- a/cvat/apps/dataset_manager/formats/yolo.py +++ b/cvat/apps/dataset_manager/formats/yolo.py @@ -5,19 +5,19 @@ import os.path as osp from glob import glob +from datumaro.components.annotation import AnnotationType +from datumaro.components.extractor import DatasetItem +from datumaro.components.project import Dataset from pyunpack import Archive from cvat.apps.dataset_manager.bindings import ( GetCVATDataExtractor, detect_dataset, + find_dataset_root, import_dm_annotations, match_dm_item, - find_dataset_root, ) from cvat.apps.dataset_manager.util import make_zip_archive -from datumaro.components.annotation import AnnotationType -from datumaro.components.extractor import DatasetItem -from datumaro.components.project import Dataset from .registry import dm_env, exporter, importer diff --git a/cvat/apps/dataset_manager/project.py b/cvat/apps/dataset_manager/project.py index 93ac651cf477..ad51370b04e1 100644 --- a/cvat/apps/dataset_manager/project.py +++ b/cvat/apps/dataset_manager/project.py @@ -6,22 +6,22 @@ import os from collections.abc import Mapping from tempfile import TemporaryDirectory -import rq from typing import Any, Callable -from datumaro.components.errors import DatasetError, DatasetImportError, DatasetNotFoundError -from django.db import transaction +import rq +from datumaro.components.errors import DatasetError, DatasetImportError, DatasetNotFoundError from django.conf import settings +from django.db import transaction +from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.engine import models from cvat.apps.engine.log import DatasetLogManager +from cvat.apps.engine.rq_job_handler import RQJobMetaField from cvat.apps.engine.serializers import DataSerializer, TaskWriteSerializer from cvat.apps.engine.task import _create_thread as create_task -from cvat.apps.engine.rq_job_handler import RQJobMetaField -from cvat.apps.dataset_manager.task import TaskAnnotation from .annotation import AnnotationIR -from .bindings import CvatDatasetNotFoundError, ProjectData, load_dataset_data, CvatImportError +from .bindings import CvatDatasetNotFoundError, CvatImportError, ProjectData, load_dataset_data from .formats.registry import make_exporter, make_importer dlogger = DatasetLogManager() diff --git a/cvat/apps/dataset_manager/task.py b/cvat/apps/dataset_manager/task.py index 83886d7e9cf1..74f035d40787 100644 --- a/cvat/apps/dataset_manager/task.py +++ b/cvat/apps/dataset_manager/task.py @@ -10,27 +10,34 @@ from enum import Enum from tempfile import TemporaryDirectory from typing import Optional, Union -from datumaro.components.errors import DatasetError, DatasetImportError, DatasetNotFoundError +from datumaro.components.errors import DatasetError, DatasetImportError, DatasetNotFoundError +from django.conf import settings from django.db import transaction from django.db.models.query import Prefetch, QuerySet -from django.conf import settings from rest_framework.exceptions import ValidationError +from cvat.apps.dataset_manager.annotation import AnnotationIR, AnnotationManager +from cvat.apps.dataset_manager.bindings import ( + CvatDatasetNotFoundError, + CvatImportError, + JobData, + TaskData, +) +from cvat.apps.dataset_manager.formats.registry import make_exporter, make_importer +from cvat.apps.dataset_manager.util import ( + add_prefetch_fields, + bulk_create, + faster_deepcopy, + get_cached, +) from cvat.apps.engine import models, serializers -from cvat.apps.engine.plugins import plugin_decorator from cvat.apps.engine.log import DatasetLogManager +from cvat.apps.engine.plugins import plugin_decorator from cvat.apps.engine.utils import take_by from cvat.apps.events.handlers import handle_annotations_change from cvat.apps.profiler import silk_profile -from cvat.apps.dataset_manager.annotation import AnnotationIR, AnnotationManager -from cvat.apps.dataset_manager.bindings import TaskData, JobData, CvatImportError, CvatDatasetNotFoundError -from cvat.apps.dataset_manager.formats.registry import make_exporter, make_importer -from cvat.apps.dataset_manager.util import ( - add_prefetch_fields, bulk_create, get_cached, faster_deepcopy -) - dlogger = DatasetLogManager() class dotdict(OrderedDict): diff --git a/cvat/apps/dataset_manager/tests/test_formats.py b/cvat/apps/dataset_manager/tests/test_formats.py index e6ba111f29f9..9dced8542c21 100644 --- a/cvat/apps/dataset_manager/tests/test_formats.py +++ b/cvat/apps/dataset_manager/tests/test_formats.py @@ -4,28 +4,33 @@ # # SPDX-License-Identifier: MIT -import numpy as np import os.path as osp import tempfile import zipfile from io import BytesIO import datumaro -from datumaro.components.dataset import Dataset, DatasetItem +import numpy as np from datumaro.components.annotation import Mask +from datumaro.components.dataset import Dataset, DatasetItem from django.contrib.auth.models import Group, User - from rest_framework import status import cvat.apps.dataset_manager as dm from cvat.apps.dataset_manager.annotation import AnnotationIR -from cvat.apps.dataset_manager.bindings import (CvatTaskOrJobDataExtractor, - TaskData, find_dataset_root) +from cvat.apps.dataset_manager.bindings import ( + CvatTaskOrJobDataExtractor, + TaskData, + find_dataset_root, +) from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.dataset_manager.util import make_zip_archive from cvat.apps.engine.models import Task from cvat.apps.engine.tests.utils import ( - get_paginated_collection, ForceLogin, generate_image_file, ApiTestBase + ApiTestBase, + ForceLogin, + generate_image_file, + get_paginated_collection, ) diff --git a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py index f3640b835bcb..7be4f0e753a4 100644 --- a/cvat/apps/dataset_manager/tests/test_rest_api_formats.py +++ b/cvat/apps/dataset_manager/tests/test_rest_api_formats.py @@ -6,11 +6,9 @@ import copy import itertools import json -import os.path as osp -import os import multiprocessing -import av -import numpy as np +import os +import os.path as osp import random import shutil import xml.etree.ElementTree as ET @@ -22,8 +20,11 @@ from tempfile import TemporaryDirectory from time import sleep from typing import Any, Callable, ClassVar, Optional, overload -from unittest.mock import MagicMock, patch, DEFAULT as MOCK_DEFAULT +from unittest.mock import DEFAULT as MOCK_DEFAULT +from unittest.mock import MagicMock, patch +import av +import numpy as np from attr import define, field from datumaro.components.dataset import Dataset from datumaro.components.operations import ExactComparator @@ -38,7 +39,7 @@ from cvat.apps.dataset_manager.util import get_export_cache_lock from cvat.apps.dataset_manager.views import clear_export_cache, export, parse_export_file_path from cvat.apps.engine.models import Task -from cvat.apps.engine.tests.utils import get_paginated_collection, ApiTestBase, ForceLogin +from cvat.apps.engine.tests.utils import ApiTestBase, ForceLogin, get_paginated_collection projects_path = osp.join(osp.dirname(__file__), 'assets', 'projects.json') with open(projects_path) as file: @@ -1451,8 +1452,8 @@ def _export(*_, task_id: int): import sys from os import replace as original_replace from os.path import exists as original_exists - from cvat.apps.dataset_manager.task import export_task as original_export_task + from cvat.apps.dataset_manager.task import export_task as original_export_task from cvat.apps.dataset_manager.views import log_exception as original_log_exception def patched_log_exception(logger=None, exc_info=True): diff --git a/cvat/apps/dataset_manager/views.py b/cvat/apps/dataset_manager/views.py index 52bc9cd15f7a..4dcd8304e43d 100644 --- a/cvat/apps/dataset_manager/views.py +++ b/cvat/apps/dataset_manager/views.py @@ -8,10 +8,10 @@ import os.path as osp import tempfile from datetime import timedelta +from os.path import exists as osp_exists import django_rq import rq -from os.path import exists as osp_exists from django.conf import settings from django.utils import timezone from rq_scheduler import Scheduler @@ -20,18 +20,20 @@ import cvat.apps.dataset_manager.task as task from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.models import Job, Project, Task -from cvat.apps.engine.utils import get_rq_lock_by_user from cvat.apps.engine.rq_job_handler import RQMeta +from cvat.apps.engine.utils import get_rq_lock_by_user from .formats.registry import EXPORT_FORMATS, IMPORT_FORMATS +from .util import EXPORT_CACHE_DIR_NAME # pylint: disable=unused-import from .util import ( LockNotAvailableError, - current_function_name, get_export_cache_lock, - get_export_cache_dir, make_export_filename, - parse_export_file_path, extend_export_file_lifetime + current_function_name, + extend_export_file_lifetime, + get_export_cache_dir, + get_export_cache_lock, + make_export_filename, + parse_export_file_path, ) -from .util import EXPORT_CACHE_DIR_NAME # pylint: disable=unused-import - slogger = ServerLogManager(__name__) diff --git a/cvat/apps/dataset_repo/migrations/0001_initial.py b/cvat/apps/dataset_repo/migrations/0001_initial.py index 2ecf9c17c9b9..fb4cf35a9672 100644 --- a/cvat/apps/dataset_repo/migrations/0001_initial.py +++ b/cvat/apps/dataset_repo/migrations/0001_initial.py @@ -1,7 +1,7 @@ # Generated by Django 2.1.3 on 2018-12-05 13:24 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/dataset_repo/migrations/0004_rename.py b/cvat/apps/dataset_repo/migrations/0004_rename.py index 9629165722d1..05072160398e 100644 --- a/cvat/apps/dataset_repo/migrations/0004_rename.py +++ b/cvat/apps/dataset_repo/migrations/0004_rename.py @@ -1,5 +1,6 @@ from django.db import migrations + def update_contenttypes_table(apps, schema_editor): content_type_model = apps.get_model('contenttypes', 'ContentType') content_type_model.objects.filter(app_label='git').update(app_label='dataset_repo') diff --git a/cvat/apps/engine/__init__.py b/cvat/apps/engine/__init__.py index 325276288f0f..f6b1f2bb9381 100644 --- a/cvat/apps/engine/__init__.py +++ b/cvat/apps/engine/__init__.py @@ -4,4 +4,4 @@ # SPDX-License-Identifier: MIT -from .schema import * # force import of declared symbols +from .schema import * # force import of declared symbols diff --git a/cvat/apps/engine/admin.py b/cvat/apps/engine/admin.py index 05e4b40a0f9b..712e67fa5582 100644 --- a/cvat/apps/engine/admin.py +++ b/cvat/apps/engine/admin.py @@ -4,8 +4,21 @@ # SPDX-License-Identifier: MIT from django.contrib import admin -from .models import Task, Segment, Job, Label, AttributeSpec, Project, \ - CloudStorage, Storage, Data, AnnotationGuide, Asset + +from .models import ( + AnnotationGuide, + Asset, + AttributeSpec, + CloudStorage, + Data, + Job, + Label, + Project, + Segment, + Storage, + Task, +) + class JobInline(admin.TabularInline): model = Job diff --git a/cvat/apps/engine/apps.py b/cvat/apps/engine/apps.py index bcad84510f5d..1cea639842c8 100644 --- a/cvat/apps/engine/apps.py +++ b/cvat/apps/engine/apps.py @@ -20,6 +20,7 @@ def ready(self): # Required to define signals in application import cvat.apps.engine.signals + # Required in order to silent "unused-import" in pyflake assert cvat.apps.engine.signals diff --git a/cvat/apps/engine/backup.py b/cvat/apps/engine/backup.py index 3c8ba5678c24..f3790427f5ba 100644 --- a/cvat/apps/engine/backup.py +++ b/cvat/apps/engine/backup.py @@ -21,37 +21,57 @@ from django.conf import settings from django.db import transaction from django.utils import timezone - from rest_framework import serializers, status +from rest_framework.exceptions import ValidationError from rest_framework.parsers import JSONParser from rest_framework.renderers import JSONRenderer from rest_framework.response import Response -from rest_framework.exceptions import ValidationError import cvat.apps.dataset_manager as dm +from cvat.apps.dataset_manager.bindings import CvatImportError +from cvat.apps.dataset_manager.views import get_export_cache_dir, log_exception from cvat.apps.engine import models +from cvat.apps.engine.cloud_provider import import_resource_from_cloud_storage +from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.serializers import (AttributeSerializer, DataSerializer, JobWriteSerializer, - LabelSerializer, AnnotationGuideWriteSerializer, AssetWriteSerializer, - LabeledDataSerializer, SegmentSerializer, SimpleJobSerializer, TaskReadSerializer, - ProjectReadSerializer, ProjectFileSerializer, TaskFileSerializer, RqIdSerializer, - ValidationParamsSerializer) -from cvat.apps.engine.utils import ( - av_scan_paths, process_failed_job, - get_rq_job_meta, import_resource_with_clean_up_after, - define_dependent_job, get_rq_lock_by_user, +from cvat.apps.engine.models import ( + DataChoice, + Location, + Project, + RequestAction, + RequestSubresource, + RequestTarget, + StorageChoice, + StorageMethodChoice, ) +from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField -from cvat.apps.engine.models import ( - StorageChoice, StorageMethodChoice, DataChoice, Project, Location, - RequestAction, RequestTarget, RequestSubresource, +from cvat.apps.engine.serializers import ( + AnnotationGuideWriteSerializer, + AssetWriteSerializer, + AttributeSerializer, + DataSerializer, + JobWriteSerializer, + LabeledDataSerializer, + LabelSerializer, + ProjectFileSerializer, + ProjectReadSerializer, + RqIdSerializer, + SegmentSerializer, + SimpleJobSerializer, + TaskFileSerializer, + TaskReadSerializer, + ValidationParamsSerializer, ) from cvat.apps.engine.task import JobFileMapping, _create_thread -from cvat.apps.engine.cloud_provider import import_resource_from_cloud_storage -from cvat.apps.engine.location import StorageType, get_location_configuration -from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export -from cvat.apps.dataset_manager.views import get_export_cache_dir, log_exception -from cvat.apps.dataset_manager.bindings import CvatImportError +from cvat.apps.engine.utils import ( + av_scan_paths, + define_dependent_job, + get_rq_job_meta, + get_rq_lock_by_user, + import_resource_with_clean_up_after, + process_failed_job, +) slogger = ServerLogManager(__name__) diff --git a/cvat/apps/engine/cloud_provider.py b/cvat/apps/engine/cloud_provider.py index b810304d73f9..06b2496ce16b 100644 --- a/cvat/apps/engine/cloud_provider.py +++ b/cvat/apps/engine/cloud_provider.py @@ -5,14 +5,14 @@ import functools import json -import os import math +import os from abc import ABC, abstractmethod from collections.abc import Iterator -from concurrent.futures import ThreadPoolExecutor, wait, FIRST_EXCEPTION +from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait from enum import Enum from io import BytesIO -from typing import Optional, Any, Callable, TypeVar +from typing import Any, Callable, Optional, TypeVar import boto3 from azure.core.exceptions import HttpResponseError, ResourceExistsError @@ -27,14 +27,14 @@ from google.cloud.exceptions import Forbidden as GoogleCloudForbidden from google.cloud.exceptions import NotFound as GoogleCloudNotFound from PIL import Image, ImageFile -from rest_framework.exceptions import (NotFound, PermissionDenied, - ValidationError) +from rest_framework.exceptions import NotFound, PermissionDenied, ValidationError from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.models import CloudProviderChoice, CredentialsTypeChoice from cvat.apps.engine.utils import get_cpu_number, take_by from cvat.utils.http import PROXIES_FOR_UNTRUSTED_URLS + class NamedBytesIO(BytesIO): @property def filename(self) -> Optional[str]: diff --git a/cvat/apps/engine/filters.py b/cvat/apps/engine/filters.py index 32355629d06d..6a80e94ad6cc 100644 --- a/cvat/apps/engine/filters.py +++ b/cvat/apps/engine/filters.py @@ -3,25 +3,25 @@ # # SPDX-License-Identifier: MIT -from collections.abc import Iterator, Iterable +import json +import operator +from collections.abc import Iterable, Iterator from functools import reduce +from textwrap import dedent from typing import Any, Optional -import operator -import json +from django.db.models import Q +from django.db.models.query import QuerySet +from django.utils.encoding import force_str +from django.utils.translation import gettext_lazy as _ from django_filters import FilterSet from django_filters import filters as djf from django_filters.filterset import BaseFilterSet from django_filters.rest_framework import DjangoFilterBackend -from django.db.models import Q -from django.db.models.query import QuerySet -from django.utils.translation import gettext_lazy as _ -from django.utils.encoding import force_str -from rest_framework.request import Request from rest_framework import filters from rest_framework.compat import coreapi, coreschema from rest_framework.exceptions import ValidationError -from textwrap import dedent +from rest_framework.request import Request DEFAULT_FILTER_FIELDS_ATTR = 'filter_fields' DEFAULT_LOOKUP_MAP_ATTR = 'lookup_fields' diff --git a/cvat/apps/engine/frame_provider.py b/cvat/apps/engine/frame_provider.py index 6b756543c7f3..1a5fd1f40ebd 100644 --- a/cvat/apps/engine/frame_provider.py +++ b/cvat/apps/engine/frame_provider.py @@ -15,15 +15,7 @@ from dataclasses import dataclass from enum import Enum, auto from io import BytesIO -from typing import ( - Any, - Callable, - Generic, - Optional, - TypeVar, - Union, - overload, -) +from typing import Any, Callable, Generic, Optional, TypeVar, Union, overload import av import cv2 diff --git a/cvat/apps/engine/handlers.py b/cvat/apps/engine/handlers.py index d686bbf0ba5c..0a831a44827b 100644 --- a/cvat/apps/engine/handlers.py +++ b/cvat/apps/engine/handlers.py @@ -4,7 +4,9 @@ from pathlib import Path from time import time + from django.conf import settings + from cvat.apps.engine.log import ServerLogManager slogger = ServerLogManager(__name__) diff --git a/cvat/apps/engine/location.py b/cvat/apps/engine/location.py index c9e216e24627..deea541f09d3 100644 --- a/cvat/apps/engine/location.py +++ b/cvat/apps/engine/location.py @@ -3,9 +3,10 @@ # SPDX-License-Identifier: MIT from enum import Enum -from typing import Any, Union, Optional +from typing import Any, Optional, Union + +from cvat.apps.engine.models import Job, Location, Project, Task -from cvat.apps.engine.models import Location, Project, Task, Job class StorageType(str, Enum): TARGET = 'target_storage' diff --git a/cvat/apps/engine/log.py b/cvat/apps/engine/log.py index 6f1740e74fd4..3cc2cecff37b 100644 --- a/cvat/apps/engine/log.py +++ b/cvat/apps/engine/log.py @@ -4,12 +4,15 @@ # SPDX-License-Identifier: MIT import logging -import sys import os.path as osp +import sys from contextlib import contextmanager -from cvat.apps.engine.utils import directory_tree + from django.conf import settings +from cvat.apps.engine.utils import directory_tree + + class _LoggerAdapter(logging.LoggerAdapter): def process(self, msg: str, kwargs): if msg_prefix := self.extra.get("msg_prefix"): diff --git a/cvat/apps/engine/management/commands/syncperiodicjobs.py b/cvat/apps/engine/management/commands/syncperiodicjobs.py index 097f468b337f..d78d3f247179 100644 --- a/cvat/apps/engine/management/commands/syncperiodicjobs.py +++ b/cvat/apps/engine/management/commands/syncperiodicjobs.py @@ -5,10 +5,10 @@ from argparse import ArgumentParser from collections import defaultdict -from django.core.management.base import BaseCommand +import django_rq from django.conf import settings +from django.core.management.base import BaseCommand -import django_rq class Command(BaseCommand): help = "Synchronize periodic jobs in Redis with the project configuration" diff --git a/cvat/apps/engine/media_extractors.py b/cvat/apps/engine/media_extractors.py index ae1c7b9f7da8..09c2ce2876de 100644 --- a/cvat/apps/engine/media_extractors.py +++ b/cvat/apps/engine/media_extractors.py @@ -5,20 +5,21 @@ from __future__ import annotations +import io +import itertools import os +import shutil +import struct import sysconfig import tempfile -import shutil import zipfile -import io -import itertools -import struct from abc import ABC, abstractmethod from bisect import bisect from collections.abc import Generator, Iterable, Iterator, Sequence from contextlib import AbstractContextManager, ExitStack, closing, contextmanager from dataclasses import dataclass from enum import IntEnum +from random import shuffle from typing import Any, Callable, Optional, Protocol, TypeVar, Union import av @@ -27,19 +28,19 @@ import av.video.stream import numpy as np from natsort import os_sorted -from pyunpack import Archive from PIL import Image, ImageFile, ImageOps -from random import shuffle -from cvat.apps.engine.utils import rotate_image -from cvat.apps.engine.models import DimensionType, SortingMethod +from pyunpack import Archive from rest_framework.exceptions import ValidationError +from cvat.apps.engine.models import DimensionType, SortingMethod +from cvat.apps.engine.utils import rotate_image + # fixes: "OSError:broken data stream" when executing line 72 while loading images downloaded from the web # see: https://stackoverflow.com/questions/42462431/oserror-broken-data-stream-when-reading-image-file ImageFile.LOAD_TRUNCATED_IMAGES = True from cvat.apps.engine.mime_types import mimetypes -from utils.dataset_manifest import VideoManifestManager, ImageManifestManager +from utils.dataset_manifest import ImageManifestManager, VideoManifestManager ORIENTATION_EXIF_TAG = 274 diff --git a/cvat/apps/engine/middleware.py b/cvat/apps/engine/middleware.py index f2b990a14b50..2e8f116f4ecd 100644 --- a/cvat/apps/engine/middleware.py +++ b/cvat/apps/engine/middleware.py @@ -4,6 +4,7 @@ from uuid import uuid4 + class RequestTrackingMiddleware: def __init__(self, get_response): self.get_response = get_response diff --git a/cvat/apps/engine/migrations/0001_release_v0_1_0.py b/cvat/apps/engine/migrations/0001_release_v0_1_0.py index 64d030cc81c6..59edc03104f4 100644 --- a/cvat/apps/engine/migrations/0001_release_v0_1_0.py +++ b/cvat/apps/engine/migrations/0001_release_v0_1_0.py @@ -5,9 +5,9 @@ # Generated by Django 2.0.3 on 2018-05-23 11:51 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0002_labeledpoints_labeledpointsattributeval_labeledpolygon_labeledpolygonattributeval_labeledpolyline_la.py b/cvat/apps/engine/migrations/0002_labeledpoints_labeledpointsattributeval_labeledpolygon_labeledpolygonattributeval_labeledpolyline_la.py index 0e7820999c38..fa3e6fe79b94 100644 --- a/cvat/apps/engine/migrations/0002_labeledpoints_labeledpointsattributeval_labeledpolygon_labeledpolygonattributeval_labeledpolyline_la.py +++ b/cvat/apps/engine/migrations/0002_labeledpoints_labeledpointsattributeval_labeledpolygon_labeledpolygonattributeval_labeledpolyline_la.py @@ -5,8 +5,8 @@ # Generated by Django 2.0.3 on 2018-05-30 09:53 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0008_auto_20180917_1424.py b/cvat/apps/engine/migrations/0008_auto_20180917_1424.py index cf6b45500d90..a32051d585e4 100644 --- a/cvat/apps/engine/migrations/0008_auto_20180917_1424.py +++ b/cvat/apps/engine/migrations/0008_auto_20180917_1424.py @@ -1,8 +1,8 @@ # Generated by Django 2.0.3 on 2018-09-17 11:24 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0011_add_task_source_and_safecharfield.py b/cvat/apps/engine/migrations/0011_add_task_source_and_safecharfield.py index bb96c1b588dd..4b168322d486 100644 --- a/cvat/apps/engine/migrations/0011_add_task_source_and_safecharfield.py +++ b/cvat/apps/engine/migrations/0011_add_task_source_and_safecharfield.py @@ -1,8 +1,9 @@ # Generated by Django 2.0.9 on 2018-10-24 10:50 -import cvat.apps.engine.models from django.db import migrations +import cvat.apps.engine.models + class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0013_auth_no_default_permissions.py b/cvat/apps/engine/migrations/0013_auth_no_default_permissions.py index bc735269eed6..2dabe07fe9a0 100644 --- a/cvat/apps/engine/migrations/0013_auth_no_default_permissions.py +++ b/cvat/apps/engine/migrations/0013_auth_no_default_permissions.py @@ -1,8 +1,8 @@ # Generated by Django 2.0.9 on 2018-11-07 12:25 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0015_db_redesign_20190217.py b/cvat/apps/engine/migrations/0015_db_redesign_20190217.py index db9589d8b807..accac35b8187 100644 --- a/cvat/apps/engine/migrations/0015_db_redesign_20190217.py +++ b/cvat/apps/engine/migrations/0015_db_redesign_20190217.py @@ -1,11 +1,13 @@ # Generated by Django 2.1.5 on 2019-02-17 19:32 -from django.conf import settings -from django.db import migrations, models import django.db.migrations.operations.special import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + import cvat.apps.engine.models + def set_segment_size(apps, schema_editor): Task = apps.get_model('engine', 'Task') for task in Task.objects.all(): diff --git a/cvat/apps/engine/migrations/0016_attribute_spec_20190217.py b/cvat/apps/engine/migrations/0016_attribute_spec_20190217.py index 27d273af2790..ac060ad69326 100644 --- a/cvat/apps/engine/migrations/0016_attribute_spec_20190217.py +++ b/cvat/apps/engine/migrations/0016_attribute_spec_20190217.py @@ -1,12 +1,15 @@ +import csv import os import re -import csv from io import StringIO -from PIL import Image -from django.db import migrations + from django.conf import settings +from django.db import migrations +from PIL import Image + from cvat.apps.engine.media_extractors import get_mime + def parse_attribute(value): match = re.match(r'^([~@])(\w+)=(\w+):(.+)?$', value) if match: diff --git a/cvat/apps/engine/migrations/0017_db_redesign_20190221.py b/cvat/apps/engine/migrations/0017_db_redesign_20190221.py index 22b7e5d28881..d30d5fa0a73a 100644 --- a/cvat/apps/engine/migrations/0017_db_redesign_20190221.py +++ b/cvat/apps/engine/migrations/0017_db_redesign_20190221.py @@ -1,11 +1,13 @@ # Generated by Django 2.1.5 on 2019-02-21 12:25 -import cvat.apps.engine.models -from django.db import migrations, models import django.db.models.deletion from django.conf import settings +from django.db import migrations, models + +import cvat.apps.engine.models from cvat.apps.dataset_manager.task import merge_table_rows as _merge_table_rows + # some modified functions to transfer annotation def _bulk_create(db_model, db_alias, objects, flt_param): if objects: diff --git a/cvat/apps/engine/migrations/0018_jobcommit.py b/cvat/apps/engine/migrations/0018_jobcommit.py index c526cb896435..b25187c50f60 100644 --- a/cvat/apps/engine/migrations/0018_jobcommit.py +++ b/cvat/apps/engine/migrations/0018_jobcommit.py @@ -1,8 +1,8 @@ # Generated by Django 2.1.7 on 2019-04-17 09:25 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0020_remove_task_flipped.py b/cvat/apps/engine/migrations/0020_remove_task_flipped.py index 7ca57e880417..7744def2b302 100644 --- a/cvat/apps/engine/migrations/0020_remove_task_flipped.py +++ b/cvat/apps/engine/migrations/0020_remove_task_flipped.py @@ -1,14 +1,15 @@ # Generated by Django 2.1.7 on 2019-06-18 11:08 -from django.db import migrations +import os +from ast import literal_eval + from django.conf import settings +from django.db import migrations +from PIL import Image -from cvat.apps.engine.models import Job, ShapeType from cvat.apps.engine.media_extractors import get_mime +from cvat.apps.engine.models import Job, ShapeType -from PIL import Image -from ast import literal_eval -import os def make_image_meta_cache(db_task): with open(db_task.get_image_meta_cache_path(), 'w') as meta_file: diff --git a/cvat/apps/engine/migrations/0022_auto_20191004_0817.py b/cvat/apps/engine/migrations/0022_auto_20191004_0817.py index b48a24f583db..6fd0ca45d8c3 100644 --- a/cvat/apps/engine/migrations/0022_auto_20191004_0817.py +++ b/cvat/apps/engine/migrations/0022_auto_20191004_0817.py @@ -1,9 +1,10 @@ # Generated by Django 2.2.3 on 2019-10-04 08:17 -import cvat.apps.engine.models +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0023_auto_20200113_1323.py b/cvat/apps/engine/migrations/0023_auto_20200113_1323.py index 4089eb1a1a66..33c586398323 100644 --- a/cvat/apps/engine/migrations/0023_auto_20200113_1323.py +++ b/cvat/apps/engine/migrations/0023_auto_20200113_1323.py @@ -1,8 +1,9 @@ # Generated by Django 2.2.8 on 2020-01-13 13:23 -import cvat.apps.engine.models from django.db import migrations +import cvat.apps.engine.models + class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0024_auto_20191023_1025.py b/cvat/apps/engine/migrations/0024_auto_20191023_1025.py index 1946e08e47e2..945879ef7552 100644 --- a/cvat/apps/engine/migrations/0024_auto_20191023_1025.py +++ b/cvat/apps/engine/migrations/0024_auto_20191023_1025.py @@ -1,24 +1,32 @@ # Generated by Django 2.2.4 on 2019-10-23 10:25 +import glob +import itertools +import multiprocessing import os import re import shutil -import glob import sys -import traceback -import itertools -import multiprocessing import time +import traceback -from django.db import migrations, models import django.db.models.deletion from django.conf import settings +from django.db import migrations, models -from cvat.apps.engine.media_extractors import (VideoReader, ArchiveReader, ZipReader, - PdfReader , ImageListReader, Mpeg4ChunkWriter, - ZipChunkWriter, ZipCompressedChunkWriter, get_mime) -from cvat.apps.engine.models import DataChoice from cvat.apps.engine.log import get_migration_logger +from cvat.apps.engine.media_extractors import ( + ArchiveReader, + ImageListReader, + Mpeg4ChunkWriter, + PdfReader, + VideoReader, + ZipChunkWriter, + ZipCompressedChunkWriter, + ZipReader, + get_mime, +) +from cvat.apps.engine.models import DataChoice MIGRATION_THREAD_COUNT = 2 diff --git a/cvat/apps/engine/migrations/0028_labelcolor.py b/cvat/apps/engine/migrations/0028_labelcolor.py index af30fbabd8d2..eda6215ecdd6 100644 --- a/cvat/apps/engine/migrations/0028_labelcolor.py +++ b/cvat/apps/engine/migrations/0028_labelcolor.py @@ -1,7 +1,9 @@ # Generated by Django 2.2.13 on 2020-08-11 11:26 from django.db import migrations, models + from cvat.apps.dataset_manager.formats.utils import get_label_color + def alter_label_colors(apps, schema_editor): Label = apps.get_model('engine', 'Label') Task = apps.get_model('engine', 'Task') diff --git a/cvat/apps/engine/migrations/0029_data_storage_method.py b/cvat/apps/engine/migrations/0029_data_storage_method.py index 1c1aa814e4cd..e5ee36f33f06 100644 --- a/cvat/apps/engine/migrations/0029_data_storage_method.py +++ b/cvat/apps/engine/migrations/0029_data_storage_method.py @@ -1,12 +1,15 @@ # Generated by Django 2.2.13 on 2020-08-13 05:49 -from cvat.apps.engine.media_extractors import _is_archive, _is_zip -import cvat.apps.engine.models +import os + from django.conf import settings from django.db import migrations, models -import os from pyunpack import Archive +import cvat.apps.engine.models +from cvat.apps.engine.media_extractors import _is_archive, _is_zip + + def unzip(apps, schema_editor): Data = apps.get_model("engine", "Data") data_q_set = Data.objects.all() diff --git a/cvat/apps/engine/migrations/0033_projects_adjastment.py b/cvat/apps/engine/migrations/0033_projects_adjastment.py index e57bd0e6c568..8af73e6d1da5 100644 --- a/cvat/apps/engine/migrations/0033_projects_adjastment.py +++ b/cvat/apps/engine/migrations/0033_projects_adjastment.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.1 on 2020-09-24 12:44 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0034_auto_20201125_1426.py b/cvat/apps/engine/migrations/0034_auto_20201125_1426.py index 311b21655b9d..d02582342893 100644 --- a/cvat/apps/engine/migrations/0034_auto_20201125_1426.py +++ b/cvat/apps/engine/migrations/0034_auto_20201125_1426.py @@ -1,9 +1,11 @@ # Generated by Django 3.1.1 on 2020-11-25 14:26 -import cvat.apps.engine.models +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + +import cvat.apps.engine.models + def create_profile(apps, schema_editor): User = apps.get_model('auth', 'User') diff --git a/cvat/apps/engine/migrations/0035_data_storage.py b/cvat/apps/engine/migrations/0035_data_storage.py index 5a8a9903784f..075d7ce38015 100644 --- a/cvat/apps/engine/migrations/0035_data_storage.py +++ b/cvat/apps/engine/migrations/0035_data_storage.py @@ -1,8 +1,9 @@ # Generated by Django 3.1.1 on 2020-12-02 06:47 -import cvat.apps.engine.models from django.db import migrations, models +import cvat.apps.engine.models + class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0036_auto_20201216_0943.py b/cvat/apps/engine/migrations/0036_auto_20201216_0943.py index 6f2fde01250f..52cb5faca2a5 100644 --- a/cvat/apps/engine/migrations/0036_auto_20201216_0943.py +++ b/cvat/apps/engine/migrations/0036_auto_20201216_0943.py @@ -1,8 +1,9 @@ # Generated by Django 3.1.1 on 2020-12-16 09:43 -import cvat.apps.engine.models -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0038_manifest.py b/cvat/apps/engine/migrations/0038_manifest.py index 002a0326c2dc..33208ad4cf19 100644 --- a/cvat/apps/engine/migrations/0038_manifest.py +++ b/cvat/apps/engine/migrations/0038_manifest.py @@ -9,9 +9,8 @@ from django.db import migrations from cvat.apps.engine.log import get_logger -from cvat.apps.engine.models import (DimensionType, StorageChoice, - StorageMethodChoice) from cvat.apps.engine.media_extractors import get_mime +from cvat.apps.engine.models import DimensionType, StorageChoice, StorageMethodChoice from utils.dataset_manifest import ImageManifestManager, VideoManifestManager MIGRATION_NAME = os.path.splitext(os.path.basename(__file__))[0] diff --git a/cvat/apps/engine/migrations/0039_auto_training.py b/cvat/apps/engine/migrations/0039_auto_training.py index a9f22ea7a03a..4594942d801e 100644 --- a/cvat/apps/engine/migrations/0039_auto_training.py +++ b/cvat/apps/engine/migrations/0039_auto_training.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.7 on 2021-04-02 13:17 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0040_cloud_storage.py b/cvat/apps/engine/migrations/0040_cloud_storage.py index c73609fd9fef..f7ecac010d19 100644 --- a/cvat/apps/engine/migrations/0040_cloud_storage.py +++ b/cvat/apps/engine/migrations/0040_cloud_storage.py @@ -1,9 +1,10 @@ # Generated by Django 3.1.8 on 2021-05-07 06:42 -import cvat.apps.engine.models +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0042_auto_20210830_1056.py b/cvat/apps/engine/migrations/0042_auto_20210830_1056.py index 7b5a496af97c..69866f2c788a 100644 --- a/cvat/apps/engine/migrations/0042_auto_20210830_1056.py +++ b/cvat/apps/engine/migrations/0042_auto_20210830_1056.py @@ -1,7 +1,7 @@ # Generated by Django 3.1.13 on 2021-08-30 10:56 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0046_data_sorting_method.py b/cvat/apps/engine/migrations/0046_data_sorting_method.py index f3880482fc33..cb58bce9ed69 100644 --- a/cvat/apps/engine/migrations/0046_data_sorting_method.py +++ b/cvat/apps/engine/migrations/0046_data_sorting_method.py @@ -1,8 +1,9 @@ # Generated by Django 3.1.13 on 2021-12-03 08:06 -import cvat.apps.engine.models from django.db import migrations, models +import cvat.apps.engine.models + class Migration(migrations.Migration): replaces = [('engine', '0045_data_sorting_method')] diff --git a/cvat/apps/engine/migrations/0047_auto_20211110_1938.py b/cvat/apps/engine/migrations/0047_auto_20211110_1938.py index 69434115f269..0113b1816c67 100644 --- a/cvat/apps/engine/migrations/0047_auto_20211110_1938.py +++ b/cvat/apps/engine/migrations/0047_auto_20211110_1938.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.8 on 2021-11-10 19:38 -import cvat.apps.engine.models -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0048_auto_20211112_1918.py b/cvat/apps/engine/migrations/0048_auto_20211112_1918.py index e1c54ab1206b..6c2106624397 100644 --- a/cvat/apps/engine/migrations/0048_auto_20211112_1918.py +++ b/cvat/apps/engine/migrations/0048_auto_20211112_1918.py @@ -1,8 +1,8 @@ # Generated by Django 3.2.8 on 2021-11-12 19:18 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0053_data_deleted_frames.py b/cvat/apps/engine/migrations/0053_data_deleted_frames.py index 8bbf49792f49..e1421a0a2c1f 100644 --- a/cvat/apps/engine/migrations/0053_data_deleted_frames.py +++ b/cvat/apps/engine/migrations/0053_data_deleted_frames.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.12 on 2022-05-20 09:21 -import cvat.apps.engine.models from django.db import migrations +import cvat.apps.engine.models + class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0054_auto_20220610_1829.py b/cvat/apps/engine/migrations/0054_auto_20220610_1829.py index 1c7ae1a802ec..25ed5b9c0617 100644 --- a/cvat/apps/engine/migrations/0054_auto_20220610_1829.py +++ b/cvat/apps/engine/migrations/0054_auto_20220610_1829.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.12 on 2022-06-10 18:29 -import cvat.apps.engine.models -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0055_jobs_directories.py b/cvat/apps/engine/migrations/0055_jobs_directories.py index ec97f2c8d3d5..89d7cd300b24 100644 --- a/cvat/apps/engine/migrations/0055_jobs_directories.py +++ b/cvat/apps/engine/migrations/0055_jobs_directories.py @@ -3,8 +3,9 @@ import os import shutil -from django.db import migrations from django.conf import settings +from django.db import migrations + from cvat.apps.engine.log import get_logger MIGRATION_NAME = os.path.splitext(os.path.basename(__file__))[0] diff --git a/cvat/apps/engine/migrations/0056_jobs_previews.py b/cvat/apps/engine/migrations/0056_jobs_previews.py index b8722018f92b..f3e6235fc780 100644 --- a/cvat/apps/engine/migrations/0056_jobs_previews.py +++ b/cvat/apps/engine/migrations/0056_jobs_previews.py @@ -2,8 +2,10 @@ import os import shutil -from django.db import migrations + from django.conf import settings +from django.db import migrations + from cvat.apps.engine.log import get_logger MIGRATION_NAME = os.path.splitext(os.path.basename(__file__))[0] diff --git a/cvat/apps/engine/migrations/0057_auto_20220726_0926.py b/cvat/apps/engine/migrations/0057_auto_20220726_0926.py index 459dbad6e783..22cd9f15e70b 100644 --- a/cvat/apps/engine/migrations/0057_auto_20220726_0926.py +++ b/cvat/apps/engine/migrations/0057_auto_20220726_0926.py @@ -1,8 +1,9 @@ # Generated by Django 3.2.14 on 2022-07-26 09:26 -import cvat.apps.engine.models -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models + +import cvat.apps.engine.models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0058_auto_20220809_1236.py b/cvat/apps/engine/migrations/0058_auto_20220809_1236.py index 8a7eb002d0af..aafb9a3bfab0 100644 --- a/cvat/apps/engine/migrations/0058_auto_20220809_1236.py +++ b/cvat/apps/engine/migrations/0058_auto_20220809_1236.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.15 on 2022-08-09 12:36 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0060_alter_label_parent.py b/cvat/apps/engine/migrations/0060_alter_label_parent.py index 5eb698343413..a5e8a8df31f3 100644 --- a/cvat/apps/engine/migrations/0060_alter_label_parent.py +++ b/cvat/apps/engine/migrations/0060_alter_label_parent.py @@ -1,7 +1,7 @@ # Generated by Django 3.2.15 on 2022-09-09 09:00 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0062_delete_previews.py b/cvat/apps/engine/migrations/0062_delete_previews.py index da986be097fb..ccf5e8f9f176 100644 --- a/cvat/apps/engine/migrations/0062_delete_previews.py +++ b/cvat/apps/engine/migrations/0062_delete_previews.py @@ -2,10 +2,12 @@ import sys import traceback -from django.db import migrations from django.conf import settings +from django.db import migrations + from cvat.apps.engine.log import get_migration_logger + def delete_previews(apps, schema_editor): migration_name = os.path.splitext(os.path.basename(__file__))[0] with get_migration_logger(migration_name) as log: diff --git a/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py b/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py index 63c167381529..97cad2c4f565 100644 --- a/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py +++ b/cvat/apps/engine/migrations/0064_delete_or_rename_wrong_labels.py @@ -1,8 +1,10 @@ import os from django.db import migrations + from cvat.apps.engine.log import get_migration_logger + def delete_or_rename_wrong_labels(apps, schema_editor): migration_name = os.path.splitext(os.path.basename(__file__))[0] with get_migration_logger(migration_name) as log: diff --git a/cvat/apps/engine/migrations/0070_add_job_type_created_date.py b/cvat/apps/engine/migrations/0070_add_job_type_created_date.py index 034a6b275ae9..62d0293245cf 100644 --- a/cvat/apps/engine/migrations/0070_add_job_type_created_date.py +++ b/cvat/apps/engine/migrations/0070_add_job_type_created_date.py @@ -1,6 +1,7 @@ -import cvat.apps.engine.models -from django.db import migrations, models import django.utils.timezone +from django.db import migrations, models + +import cvat.apps.engine.models def add_created_date_to_existing_jobs(apps, schema_editor): diff --git a/cvat/apps/engine/migrations/0071_annotationguide_asset.py b/cvat/apps/engine/migrations/0071_annotationguide_asset.py index 1060c4576aba..a6b50c50861b 100644 --- a/cvat/apps/engine/migrations/0071_annotationguide_asset.py +++ b/cvat/apps/engine/migrations/0071_annotationguide_asset.py @@ -1,9 +1,10 @@ # Generated by Django 3.2.18 on 2023-06-13 13:14 +import uuid + +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion -import uuid class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0072_alter_issue_updated_date.py b/cvat/apps/engine/migrations/0072_alter_issue_updated_date.py index 4c549be10aa5..344036d12f65 100644 --- a/cvat/apps/engine/migrations/0072_alter_issue_updated_date.py +++ b/cvat/apps/engine/migrations/0072_alter_issue_updated_date.py @@ -2,6 +2,7 @@ from django.db import migrations, models + def forwards_func(apps, schema_editor): Issue = apps.get_model("engine", "Issue") diff --git a/cvat/apps/engine/migrations/0076_remove_storages_that_refer_to_deleted_cloud_storages.py b/cvat/apps/engine/migrations/0076_remove_storages_that_refer_to_deleted_cloud_storages.py index 50c1461319a7..41c902bb2500 100644 --- a/cvat/apps/engine/migrations/0076_remove_storages_that_refer_to_deleted_cloud_storages.py +++ b/cvat/apps/engine/migrations/0076_remove_storages_that_refer_to_deleted_cloud_storages.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.6 on 2023-11-17 10:10 from django.db import migrations, models + from cvat.apps.engine.models import Location diff --git a/cvat/apps/engine/migrations/0077_auto_20231121_1952.py b/cvat/apps/engine/migrations/0077_auto_20231121_1952.py index 8b5c3648e068..831e83c8712a 100644 --- a/cvat/apps/engine/migrations/0077_auto_20231121_1952.py +++ b/cvat/apps/engine/migrations/0077_auto_20231121_1952.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.6 on 2023-11-21 19:52 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0079_alter_labeledimageattributeval_image_and_more.py b/cvat/apps/engine/migrations/0079_alter_labeledimageattributeval_image_and_more.py index ccafa6086b5e..58921bc97c92 100644 --- a/cvat/apps/engine/migrations/0079_alter_labeledimageattributeval_image_and_more.py +++ b/cvat/apps/engine/migrations/0079_alter_labeledimageattributeval_image_and_more.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.13 on 2024-07-09 11:08 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0080_alter_trackedshape_track.py b/cvat/apps/engine/migrations/0080_alter_trackedshape_track.py index d5997d15ff91..8266dbf4ba38 100644 --- a/cvat/apps/engine/migrations/0080_alter_trackedshape_track.py +++ b/cvat/apps/engine/migrations/0080_alter_trackedshape_track.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.13 on 2024-07-12 19:01 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0082_alter_labeledimage_job_and_more.py b/cvat/apps/engine/migrations/0082_alter_labeledimage_job_and_more.py index 50b91829b213..ecbc9d76f60d 100644 --- a/cvat/apps/engine/migrations/0082_alter_labeledimage_job_and_more.py +++ b/cvat/apps/engine/migrations/0082_alter_labeledimage_job_and_more.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.14 on 2024-07-22 07:27 -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/cvat/apps/engine/migrations/0085_segment_chunks_updated_date.py b/cvat/apps/engine/migrations/0085_segment_chunks_updated_date.py index 52342d7db774..6fed44b22a6a 100644 --- a/cvat/apps/engine/migrations/0085_segment_chunks_updated_date.py +++ b/cvat/apps/engine/migrations/0085_segment_chunks_updated_date.py @@ -1,6 +1,7 @@ # Generated by Django 4.2.15 on 2024-09-25 13:52 from datetime import datetime + from django.db import migrations, models diff --git a/cvat/apps/engine/mime_types.py b/cvat/apps/engine/mime_types.py index 8e70c5cc4193..fad18ba6b6f8 100644 --- a/cvat/apps/engine/mime_types.py +++ b/cvat/apps/engine/mime_types.py @@ -2,9 +2,8 @@ # # SPDX-License-Identifier: MIT -import os import mimetypes - +import os _SCRIPT_DIR = os.path.realpath(os.path.dirname(__file__)) MEDIA_MIMETYPES_FILES = [ diff --git a/cvat/apps/engine/mixins.py b/cvat/apps/engine/mixins.py index 39f50ed31db4..9e69ffdd5ccb 100644 --- a/cvat/apps/engine/mixins.py +++ b/cvat/apps/engine/mixins.py @@ -12,9 +12,9 @@ from dataclasses import asdict, dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from unittest import mock from textwrap import dedent -from typing import Optional, Callable, Any +from typing import Any, Callable, Optional +from unittest import mock from urllib.parse import urljoin import django_rq @@ -22,20 +22,18 @@ from django.conf import settings from django.http import HttpRequest from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import (OpenApiParameter, OpenApiResponse, - extend_schema) +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema from rest_framework import mixins, status -from rest_framework.decorators import action from rest_framework.authentication import SessionAuthentication +from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.views import APIView -from cvat.apps.engine.background import (BackupExportManager, - DatasetExportManager) +from cvat.apps.engine.background import BackupExportManager, DatasetExportManager from cvat.apps.engine.handlers import clear_import_cache from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.models import Location, RequestAction, RequestTarget, RequestSubresource +from cvat.apps.engine.models import Location, RequestAction, RequestSubresource, RequestTarget from cvat.apps.engine.rq_job_handler import RQId from cvat.apps.engine.serializers import DataSerializer, RqIdSerializer from cvat.apps.engine.utils import is_dataset_export diff --git a/cvat/apps/engine/pagination.py b/cvat/apps/engine/pagination.py index 2bb417f5c0d1..6a1dd499b893 100644 --- a/cvat/apps/engine/pagination.py +++ b/cvat/apps/engine/pagination.py @@ -3,8 +3,10 @@ # SPDX-License-Identifier: MIT import sys + from rest_framework.pagination import PageNumberPagination + class CustomPagination(PageNumberPagination): page_size_query_param = "page_size" diff --git a/cvat/apps/engine/parsers.py b/cvat/apps/engine/parsers.py index d0cecc4b02d0..03b4ebd45da8 100644 --- a/cvat/apps/engine/parsers.py +++ b/cvat/apps/engine/parsers.py @@ -4,6 +4,7 @@ from rest_framework.parsers import BaseParser + class TusUploadParser(BaseParser): # The media type is sent by TUS protocol (tus.io) for uploading files media_type = 'application/offset+octet-stream' diff --git a/cvat/apps/engine/permissions.py b/cvat/apps/engine/permissions.py index c5ddd4799c4c..a180410142cd 100644 --- a/cvat/apps/engine/permissions.py +++ b/cvat/apps/engine/permissions.py @@ -7,20 +7,23 @@ from collections.abc import Sequence from typing import Any, Optional, Union, cast -from django.shortcuts import get_object_or_404 from django.conf import settings - -from rest_framework.exceptions import ValidationError, PermissionDenied +from django.shortcuts import get_object_or_404 +from rest_framework.exceptions import PermissionDenied, ValidationError from rq.job import Job as RQJob from cvat.apps.engine.rq_job_handler import is_rq_job_owner +from cvat.apps.engine.utils import is_dataset_export from cvat.apps.iam.permissions import ( - OpenPolicyAgentPermission, StrEnum, get_iam_context, get_membership + OpenPolicyAgentPermission, + StrEnum, + get_iam_context, + get_membership, ) from cvat.apps.organizations.models import Organization from .models import AnnotationGuide, CloudStorage, Issue, Job, Label, Project, Task -from cvat.apps.engine.utils import is_dataset_export + def _get_key(d: dict[str, Any], key_path: Union[str, Sequence[str]]) -> Optional[Any]: """ diff --git a/cvat/apps/engine/renderers.py b/cvat/apps/engine/renderers.py index f56eb4d39808..542a322048ed 100644 --- a/cvat/apps/engine/renderers.py +++ b/cvat/apps/engine/renderers.py @@ -4,5 +4,6 @@ from rest_framework.renderers import JSONRenderer + class CVATAPIRenderer(JSONRenderer): media_type = 'application/vnd.cvat+json' diff --git a/cvat/apps/engine/rq_job_handler.py b/cvat/apps/engine/rq_job_handler.py index c5b31336ecdc..b4f146197afc 100644 --- a/cvat/apps/engine/rq_job_handler.py +++ b/cvat/apps/engine/rq_job_handler.py @@ -4,13 +4,14 @@ from __future__ import annotations -import attrs - -from typing import Optional, Union, Any +from typing import Any, Optional, Union from uuid import UUID + +import attrs from rq.job import Job as RQJob -from .models import RequestAction, RequestTarget, RequestSubresource +from .models import RequestAction, RequestSubresource, RequestTarget + class RQMeta: @staticmethod diff --git a/cvat/apps/engine/serializers.py b/cvat/apps/engine/serializers.py index 9f772cd24e6d..395bf59108b9 100644 --- a/cvat/apps/engine/serializers.py +++ b/cvat/apps/engine/serializers.py @@ -5,48 +5,55 @@ from __future__ import annotations +import os +import re +import shutil +import string +import textwrap +import warnings from collections import OrderedDict from collections.abc import Iterable, Sequence from contextlib import closing -import warnings from copy import copy from datetime import timedelta from decimal import Decimal from inspect import isclass -import os -import re -import shutil -import string from tempfile import NamedTemporaryFile -import textwrap from typing import Any, Optional, Union import django_rq +import rq.defaults as rq_defaults from django.conf import settings -from django.contrib.auth.models import User, Group +from django.contrib.auth.models import Group, User from django.db import transaction -from django.db.models import prefetch_related_objects, Prefetch +from django.db.models import Prefetch, prefetch_related_objects from django.utils import timezone +from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer from numpy import random -from rest_framework import serializers, exceptions -import rq.defaults as rq_defaults -from rq.job import Job as RQJob, JobStatus as RQJobStatus +from rest_framework import exceptions, serializers +from rq.job import Job as RQJob +from rq.job import JobStatus as RQJobStatus from cvat.apps.dataset_manager.formats.utils import get_label_color from cvat.apps.engine import field_validation, models -from cvat.apps.engine.frame_provider import TaskFrameProvider, FrameQuality -from cvat.apps.engine.cloud_provider import get_cloud_storage_instance, Credentials, Status +from cvat.apps.engine.cloud_provider import Credentials, Status, get_cloud_storage_instance +from cvat.apps.engine.frame_provider import FrameQuality, TaskFrameProvider from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.permissions import TaskPermission +from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.task_validation import HoneypotFrameSelector -from cvat.apps.engine.rq_job_handler import RQJobMetaField, RQId from cvat.apps.engine.utils import ( - format_list, grouped, parse_exception_message, CvatChunkTimestampMismatchError, - parse_specific_attributes, build_field_filter_params, get_list_view_name, reverse, take_by + CvatChunkTimestampMismatchError, + build_field_filter_params, + format_list, + get_list_view_name, + grouped, + parse_exception_message, + parse_specific_attributes, + reverse, + take_by, ) -from drf_spectacular.utils import OpenApiExample, extend_schema_field, extend_schema_serializer - slogger = ServerLogManager(__name__) class WriteOnceMixin: @@ -996,7 +1003,10 @@ def validate(self, attrs): @transaction.atomic def update(self, instance: models.Job, validated_data: dict[str, Any]) -> models.Job: from cvat.apps.engine.cache import ( - MediaCache, Callback, enqueue_create_chunk_job, wait_for_rq_job + Callback, + MediaCache, + enqueue_create_chunk_job, + wait_for_rq_job, ) from cvat.apps.engine.frame_provider import JobFrameProvider diff --git a/cvat/apps/engine/signals.py b/cvat/apps/engine/signals.py index 3a964d90c2cc..456c6f228081 100644 --- a/cvat/apps/engine/signals.py +++ b/cvat/apps/engine/signals.py @@ -11,8 +11,7 @@ from django.db.models.signals import m2m_changed, post_delete, post_save from django.dispatch import receiver -from .models import CloudStorage, Data, Job, Profile, Project, StatusChoice, Task, Asset - +from .models import Asset, CloudStorage, Data, Job, Profile, Project, StatusChoice, Task # TODO: need to log any problems reported by shutil.rmtree when the new # analytics feature is available. Now the log system can write information diff --git a/cvat/apps/engine/task.py b/cvat/apps/engine/task.py index 0f36674299fc..7aa92acba2fd 100644 --- a/cvat/apps/engine/task.py +++ b/cvat/apps/engine/task.py @@ -4,24 +4,24 @@ # SPDX-License-Identifier: MIT import concurrent.futures -import itertools import fnmatch +import itertools import os import re -import rq import shutil from collections.abc import Iterator, Sequence -from copy import deepcopy from contextlib import closing +from copy import deepcopy from datetime import datetime, timezone from pathlib import Path from typing import Any, NamedTuple, Optional, Union from urllib import parse as urlparse from urllib import request as urlrequest -import av import attrs +import av import django_rq +import rq from django.conf import settings from django.db import transaction from django.forms.models import model_to_dict @@ -29,25 +29,39 @@ from rest_framework.serializers import ValidationError from cvat.apps.engine import models -from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.frame_provider import TaskFrameProvider +from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.media_extractors import ( - MEDIA_TYPES, CachingMediaIterator, IMediaReader, ImageListReader, - Mpeg4ChunkWriter, Mpeg4CompressedChunkWriter, RandomAccessIterator, - ValidateDimension, ZipChunkWriter, ZipCompressedChunkWriter, get_mime, sort, + MEDIA_TYPES, + CachingMediaIterator, + ImageListReader, + IMediaReader, + Mpeg4ChunkWriter, + Mpeg4CompressedChunkWriter, + RandomAccessIterator, + ValidateDimension, + ZipChunkWriter, + ZipCompressedChunkWriter, + get_mime, load_image, + sort, ) from cvat.apps.engine.models import RequestAction, RequestTarget -from cvat.apps.engine.utils import ( - av_scan_paths, format_list, get_rq_job_meta, - define_dependent_job, get_rq_lock_by_user, take_by -) from cvat.apps.engine.rq_job_handler import RQId from cvat.apps.engine.task_validation import HoneypotFrameSelector -from cvat.utils.http import make_requests_session, PROXIES_FOR_UNTRUSTED_URLS +from cvat.apps.engine.utils import ( + av_scan_paths, + define_dependent_job, + format_list, + get_rq_job_meta, + get_rq_lock_by_user, + take_by, +) +from cvat.utils.http import PROXIES_FOR_UNTRUSTED_URLS, make_requests_session from utils.dataset_manifest import ImageManifestManager, VideoManifestManager, is_manifest from utils.dataset_manifest.core import VideoManifestValidator, is_dataset_manifest from utils.dataset_manifest.utils import detect_related_images + from .cloud_provider import db_storage_to_storage_instance slogger = ServerLogManager(__name__) diff --git a/cvat/apps/engine/tests/test_lazy_list.py b/cvat/apps/engine/tests/test_lazy_list.py index 6ba4b07dd38f..2a021f89b94a 100644 --- a/cvat/apps/engine/tests/test_lazy_list.py +++ b/cvat/apps/engine/tests/test_lazy_list.py @@ -1,9 +1,9 @@ -import unittest import copy import pickle +import unittest from typing import TypeVar -from cvat.apps.engine.lazy_list import LazyList +from cvat.apps.engine.lazy_list import LazyList T = TypeVar('T') diff --git a/cvat/apps/engine/tests/test_rest_api.py b/cvat/apps/engine/tests/test_rest_api.py index b0c5500eda4c..3a2f5d6e8a82 100644 --- a/cvat/apps/engine/tests/test_rest_api.py +++ b/cvat/apps/engine/tests/test_rest_api.py @@ -3,10 +3,10 @@ # # SPDX-License-Identifier: MIT -from contextlib import ExitStack -from datetime import timedelta +import copy import io -from itertools import product +import json +import logging import os import random import shutil @@ -15,40 +15,56 @@ import xml.etree.ElementTree as ET import zipfile from collections import defaultdict +from contextlib import ExitStack +from datetime import timedelta from enum import Enum from glob import glob from io import BytesIO, IOBase -from unittest import mock +from itertools import product from time import sleep -import logging -import copy -import json +from unittest import mock import av import django_rq import numpy as np -from pdf2image import convert_from_bytes -from pyunpack import Archive from django.conf import settings from django.contrib.auth.models import Group, User from django.http import HttpResponse +from pdf2image import convert_from_bytes from PIL import Image from pycocotools import coco as coco_loader +from pyunpack import Archive from rest_framework import status from rest_framework.test import APIClient from cvat.apps.dataset_manager.tests.utils import TestDir from cvat.apps.dataset_manager.util import current_function_name -from cvat.apps.engine.models import (AttributeSpec, AttributeType, Data, Job, - Project, Segment, StageChoice, StatusChoice, Task, Label, StorageMethodChoice, - StorageChoice, DimensionType, SortingMethod) from cvat.apps.engine.media_extractors import ValidateDimension, sort -from cvat.apps.engine.tests.utils import get_paginated_collection +from cvat.apps.engine.models import ( + AttributeSpec, + AttributeType, + Data, + DimensionType, + Job, + Label, + Project, + Segment, + SortingMethod, + StageChoice, + StatusChoice, + StorageChoice, + StorageMethodChoice, + Task, +) +from cvat.apps.engine.tests.utils import ( + ApiTestBase, + ForceLogin, + generate_image_file, + generate_video_file, + get_paginated_collection, +) from utils.dataset_manifest import ImageManifestManager, VideoManifestManager -from cvat.apps.engine.tests.utils import (ApiTestBase, ForceLogin, - generate_image_file, generate_video_file) - #suppress av warnings logging.getLogger('libav').setLevel(logging.ERROR) diff --git a/cvat/apps/engine/tests/test_rest_api_3D.py b/cvat/apps/engine/tests/test_rest_api_3D.py index 67791c3c113c..087448c90dd2 100644 --- a/cvat/apps/engine/tests/test_rest_api_3D.py +++ b/cvat/apps/engine/tests/test_rest_api_3D.py @@ -4,7 +4,9 @@ # SPDX-License-Identifier: MIT +import copy import io +import itertools import os import os.path as osp import tempfile @@ -13,18 +15,15 @@ from collections import defaultdict from glob import glob from io import BytesIO -import copy from shutil import copyfile -import itertools from django.contrib.auth.models import Group, User from rest_framework import status +from cvat.apps.dataset_manager.task import TaskAnnotation from cvat.apps.dataset_manager.tests.utils import TestDir from cvat.apps.engine.media_extractors import ValidateDimension -from cvat.apps.dataset_manager.task import TaskAnnotation - -from cvat.apps.engine.tests.utils import get_paginated_collection, ApiTestBase, ForceLogin +from cvat.apps.engine.tests.utils import ApiTestBase, ForceLogin, get_paginated_collection CREATE_ACTION = "create" UPDATE_ACTION = "update" diff --git a/cvat/apps/engine/tests/utils.py b/cvat/apps/engine/tests/utils.py index 910323cac1f7..09fd850b2c19 100644 --- a/cvat/apps/engine/tests/utils.py +++ b/cvat/apps/engine/tests/utils.py @@ -2,22 +2,22 @@ # # SPDX-License-Identifier: MIT +import itertools +import logging +import os from collections.abc import Iterator, Sequence from contextlib import contextmanager from io import BytesIO from typing import Any, Callable, TypeVar -import itertools -import logging -import os +import av +import django_rq +import numpy as np from django.conf import settings from django.core.cache import caches from django.http.response import HttpResponse from PIL import Image from rest_framework.test import APITestCase -import av -import django_rq -import numpy as np T = TypeVar('T') diff --git a/cvat/apps/engine/urls.py b/cvat/apps/engine/urls.py index 1755197ebcdf..1380ae5f7961 100644 --- a/cvat/apps/engine/urls.py +++ b/cvat/apps/engine/urls.py @@ -3,14 +3,13 @@ # # SPDX-License-Identifier: MIT -from django.urls import path, include -from . import views -from rest_framework import routers - -from django.views.generic import RedirectView from django.conf import settings - +from django.urls import include, path +from django.views.generic import RedirectView from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView +from rest_framework import routers + +from . import views router = routers.DefaultRouter(trailing_slash=False) router.register('projects', views.ProjectViewSet) diff --git a/cvat/apps/engine/utils.py b/cvat/apps/engine/utils.py index dd4533538f5a..b3e3d48f69d6 100644 --- a/cvat/apps/engine/utils.py +++ b/cvat/apps/engine/utils.py @@ -4,42 +4,39 @@ # SPDX-License-Identifier: MIT import ast -from itertools import islice -import cv2 as cv -from collections import namedtuple -from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence import hashlib import importlib +import logging +import os +import platform +import re +import subprocess import sys import traceback -from contextlib import suppress, nullcontext -from typing import Any, Callable, Optional, TypeVar, Union -import subprocess -import os import urllib.parse -import re -import logging -import platform +from collections import namedtuple +from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence +from contextlib import nullcontext, suppress +from itertools import islice +from multiprocessing import cpu_count +from pathlib import Path +from typing import Any, Callable, Optional, TypeVar, Union +import cv2 as cv from attr.converters import to_bool +from av import VideoFrame from datumaro.util.os_util import walk -from rq.job import Job, Dependency -from django_rq.queues import DjangoRQ -from pathlib import Path - +from django.conf import settings +from django.core.exceptions import ValidationError from django.http.request import HttpRequest from django.utils import timezone from django.utils.http import urlencode -from rest_framework.reverse import reverse as _reverse - -from av import VideoFrame -from PIL import Image -from multiprocessing import cpu_count - -from django.core.exceptions import ValidationError +from django_rq.queues import DjangoRQ from django_sendfile import sendfile as _sendfile -from django.conf import settings +from PIL import Image from redis.lock import Lock +from rest_framework.reverse import reverse as _reverse +from rq.job import Dependency, Job Import = namedtuple("Import", ["module", "name", "alias"]) @@ -230,8 +227,8 @@ def get_rq_job_meta( result_url: Optional[str] = None, ): # to prevent circular import - from cvat.apps.webhooks.signals import project_id, organization_id - from cvat.apps.events.handlers import task_id, job_id, organization_slug + from cvat.apps.events.handlers import job_id, organization_slug, task_id + from cvat.apps.webhooks.signals import organization_id, project_id oid = organization_id(db_obj) oslug = organization_slug(db_obj) diff --git a/cvat/apps/engine/view_utils.py b/cvat/apps/engine/view_utils.py index 6f5dc298a7b6..dbac90720b43 100644 --- a/cvat/apps/engine/view_utils.py +++ b/cvat/apps/engine/view_utils.py @@ -9,11 +9,11 @@ from django.db.models.query import QuerySet from django.http.request import HttpRequest from django.http.response import HttpResponse +from drf_spectacular.utils import extend_schema from rest_framework.decorators import action from rest_framework.response import Response from rest_framework.serializers import Serializer from rest_framework.viewsets import GenericViewSet -from drf_spectacular.utils import extend_schema from cvat.apps.engine.mixins import UploadMixin from cvat.apps.engine.parsers import TusUploadParser diff --git a/cvat/apps/engine/views.py b/cvat/apps/engine/views.py index eb39f6732c18..6b70836d53a1 100644 --- a/cvat/apps/engine/views.py +++ b/cvat/apps/engine/views.py @@ -13,105 +13,173 @@ import traceback import zlib from abc import ABCMeta, abstractmethod -from contextlib import suppress -from PIL import Image -from types import SimpleNamespace -from typing import Optional, Any, Union, cast, Callable from collections import namedtuple -from collections.abc import Mapping, Iterable +from collections.abc import Iterable, Mapping +from contextlib import suppress from copy import copy from datetime import datetime -from redis.exceptions import ConnectionError as RedisConnectionError +from pathlib import Path from tempfile import NamedTemporaryFile +from types import SimpleNamespace +from typing import Any, Callable, Optional, Union, cast import django_rq from attr.converters import to_bool from django.conf import settings from django.contrib.auth.models import User -from django.db import IntegrityError, transaction +from django.db import IntegrityError from django.db import models as django_models +from django.db import transaction from django.db.models.query import Prefetch -from django.http import HttpResponse, HttpRequest, HttpResponseNotFound, HttpResponseBadRequest +from django.http import HttpRequest, HttpResponse, HttpResponseBadRequest, HttpResponseNotFound from django.utils import timezone from django.utils.decorators import method_decorator from django.views.decorators.cache import never_cache from django_rq.queues import DjangoRQ - from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import ( - OpenApiExample, OpenApiParameter, OpenApiResponse, PolymorphicProxySerializer, - extend_schema_view, extend_schema + OpenApiExample, + OpenApiParameter, + OpenApiResponse, + PolymorphicProxySerializer, + extend_schema, + extend_schema_view, ) - -from pathlib import Path +from PIL import Image +from redis.exceptions import ConnectionError as RedisConnectionError from rest_framework import mixins, serializers, status, viewsets from rest_framework.decorators import action -from rest_framework.exceptions import APIException, NotFound, ValidationError, PermissionDenied +from rest_framework.exceptions import APIException, NotFound, PermissionDenied, ValidationError from rest_framework.parsers import MultiPartParser from rest_framework.permissions import SAFE_METHODS from rest_framework.response import Response from rest_framework.settings import api_settings - -from rq.job import Job as RQJob, JobStatus as RQJobStatus +from rq.job import Job as RQJob +from rq.job import JobStatus as RQJobStatus import cvat.apps.dataset_manager as dm import cvat.apps.dataset_manager.views # pylint: disable=unused-import -from cvat.apps.engine.cloud_provider import db_storage_to_storage_instance, import_resource_from_cloud_storage -from cvat.apps.events.handlers import handle_dataset_import from cvat.apps.dataset_manager.bindings import CvatImportError from cvat.apps.dataset_manager.serializers import DatasetFormatsSerializer +from cvat.apps.engine import backup +from cvat.apps.engine.cache import CvatChunkTimestampMismatchError, LockError, MediaCache +from cvat.apps.engine.cloud_provider import ( + db_storage_to_storage_instance, + import_resource_from_cloud_storage, +) +from cvat.apps.engine.filters import ( + NonModelJsonLogicFilter, + NonModelOrderingFilter, + NonModelSimpleFilter, +) from cvat.apps.engine.frame_provider import ( - DataWithMeta, IFrameProvider, TaskFrameProvider, JobFrameProvider, FrameQuality + DataWithMeta, + FrameQuality, + IFrameProvider, + JobFrameProvider, + TaskFrameProvider, ) -from cvat.apps.engine.filters import NonModelSimpleFilter, NonModelOrderingFilter, NonModelJsonLogicFilter +from cvat.apps.engine.location import StorageType, get_location_configuration from cvat.apps.engine.media_extractors import get_mime -from cvat.apps.engine.permissions import AnnotationGuidePermission, get_iam_context +from cvat.apps.engine.mixins import ( + BackupMixin, + CsrfWorkaroundMixin, + DatasetMixin, + PartialUpdateModelMixin, + UploadMixin, +) +from cvat.apps.engine.models import AnnotationGuide, Asset, ClientFile, CloudProviderChoice +from cvat.apps.engine.models import CloudStorage as CloudStorageModel from cvat.apps.engine.models import ( - ClientFile, Job, JobType, Label, Task, Project, Issue, Data, - Comment, StorageMethodChoice, StorageChoice, - CloudProviderChoice, Location, CloudStorage as CloudStorageModel, - Asset, AnnotationGuide, RequestStatus, RequestAction, RequestTarget, RequestSubresource + Comment, + Data, + Issue, + Job, + JobType, + Label, + Location, + Project, + RequestAction, + RequestStatus, + RequestSubresource, + RequestTarget, + StorageChoice, + StorageMethodChoice, + Task, +) +from cvat.apps.engine.permissions import ( + AnnotationGuidePermission, + CloudStoragePermission, + CommentPermission, + IssuePermission, + JobPermission, + LabelPermission, + ProjectPermission, + TaskPermission, + UserPermission, + get_cloud_storage_for_import_or_export, + get_iam_context, ) +from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField, is_rq_job_owner from cvat.apps.engine.serializers import ( - AboutSerializer, AnnotationFileSerializer, BasicUserSerializer, - DataMetaReadSerializer, DataMetaWriteSerializer, DataSerializer, FileInfoSerializer, - JobDataMetaWriteSerializer, JobReadSerializer, JobWriteSerializer, - JobValidationLayoutReadSerializer, JobValidationLayoutWriteSerializer, - LabelSerializer, LabeledDataSerializer, - ProjectReadSerializer, ProjectWriteSerializer, - RqStatusSerializer, TaskReadSerializer, TaskValidationLayoutReadSerializer, TaskValidationLayoutWriteSerializer, TaskWriteSerializer, - UserSerializer, PluginsSerializer, IssueReadSerializer, - AnnotationGuideReadSerializer, AnnotationGuideWriteSerializer, - AssetReadSerializer, AssetWriteSerializer, - IssueWriteSerializer, CommentReadSerializer, CommentWriteSerializer, CloudStorageWriteSerializer, - CloudStorageReadSerializer, DatasetFileSerializer, - ProjectFileSerializer, TaskFileSerializer, RqIdSerializer, CloudStorageContentSerializer, + AboutSerializer, + AnnotationFileSerializer, + AnnotationGuideReadSerializer, + AnnotationGuideWriteSerializer, + AssetReadSerializer, + AssetWriteSerializer, + BasicUserSerializer, + CloudStorageContentSerializer, + CloudStorageReadSerializer, + CloudStorageWriteSerializer, + CommentReadSerializer, + CommentWriteSerializer, + DataMetaReadSerializer, + DataMetaWriteSerializer, + DataSerializer, + DatasetFileSerializer, + FileInfoSerializer, + IssueReadSerializer, + IssueWriteSerializer, + JobDataMetaWriteSerializer, + JobReadSerializer, + JobValidationLayoutReadSerializer, + JobValidationLayoutWriteSerializer, + JobWriteSerializer, + LabeledDataSerializer, + LabelSerializer, + PluginsSerializer, + ProjectFileSerializer, + ProjectReadSerializer, + ProjectWriteSerializer, RequestSerializer, + RqIdSerializer, + RqStatusSerializer, + TaskFileSerializer, + TaskReadSerializer, + TaskValidationLayoutReadSerializer, + TaskValidationLayoutWriteSerializer, + TaskWriteSerializer, + UserSerializer, ) -from cvat.apps.engine.permissions import get_cloud_storage_for_import_or_export - -from utils.dataset_manifest import ImageManifestManager from cvat.apps.engine.utils import ( - av_scan_paths, process_failed_job, - parse_exception_message, get_rq_job_meta, - import_resource_with_clean_up_after, sendfile, define_dependent_job, get_rq_lock_by_user, -) -from cvat.apps.engine.rq_job_handler import RQId, is_rq_job_owner, RQJobMetaField -from cvat.apps.engine import backup -from cvat.apps.engine.mixins import ( - PartialUpdateModelMixin, UploadMixin, DatasetMixin, BackupMixin, CsrfWorkaroundMixin + av_scan_paths, + define_dependent_job, + get_rq_job_meta, + get_rq_lock_by_user, + import_resource_with_clean_up_after, + parse_exception_message, + process_failed_job, + sendfile, ) -from cvat.apps.engine.location import get_location_configuration, StorageType +from cvat.apps.engine.view_utils import tus_chunk_action +from cvat.apps.events.handlers import handle_dataset_import +from cvat.apps.iam.filters import ORGANIZATION_OPEN_API_PARAMETERS +from cvat.apps.iam.permissions import IsAuthenticatedOrReadPublicResource, PolicyEnforcer +from utils.dataset_manifest import ImageManifestManager from . import models, task from .log import ServerLogManager -from cvat.apps.iam.filters import ORGANIZATION_OPEN_API_PARAMETERS -from cvat.apps.iam.permissions import PolicyEnforcer, IsAuthenticatedOrReadPublicResource -from cvat.apps.engine.cache import MediaCache, CvatChunkTimestampMismatchError, LockError -from cvat.apps.engine.permissions import (CloudStoragePermission, - CommentPermission, IssuePermission, JobPermission, LabelPermission, ProjectPermission, - TaskPermission, UserPermission) -from cvat.apps.engine.view_utils import tus_chunk_action slogger = ServerLogManager(__name__) diff --git a/cvat/apps/events/apps.py b/cvat/apps/events/apps.py index f700758ad204..17c42e754f1b 100644 --- a/cvat/apps/events/apps.py +++ b/cvat/apps/events/apps.py @@ -9,7 +9,7 @@ class EventsConfig(AppConfig): name = 'cvat.apps.events' def ready(self): - from . import signals # pylint: disable=unused-import - from cvat.apps.iam.permissions import load_app_permissions load_app_permissions(self) + + from . import signals # pylint: disable=unused-import diff --git a/cvat/apps/events/cache.py b/cvat/apps/events/cache.py index 30d1e67b8fc1..168211a1e24f 100644 --- a/cvat/apps/events/cache.py +++ b/cvat/apps/events/cache.py @@ -6,7 +6,7 @@ class DeleteCache(): def __init__(self, cache_id): - from cvat.apps.engine.models import Task, Job, Issue, Comment + from cvat.apps.engine.models import Comment, Issue, Job, Task self._cache = _caches.setdefault(cache_id, { Task: {}, Job: {}, diff --git a/cvat/apps/events/event.py b/cvat/apps/events/event.py index a4afff968549..ebb00496b36c 100644 --- a/cvat/apps/events/event.py +++ b/cvat/apps/events/event.py @@ -2,14 +2,15 @@ # # SPDX-License-Identifier: MIT -from rest_framework.renderers import JSONRenderer from datetime import datetime, timezone from typing import Optional from django.db import transaction +from rest_framework.renderers import JSONRenderer from cvat.apps.engine.log import vlogger + def event_scope(action, resource): return f"{action}:{resource}" diff --git a/cvat/apps/events/export.py b/cvat/apps/events/export.py index 9225f1141162..aa70fcdc066e 100644 --- a/cvat/apps/events/export.py +++ b/cvat/apps/events/export.py @@ -2,25 +2,23 @@ # # SPDX-License-Identifier: MIT -from logging import Logger -import os import csv -from datetime import datetime, timedelta, timezone -from dateutil import parser +import os import uuid +from datetime import datetime, timedelta, timezone +from logging import Logger +import clickhouse_connect import django_rq +from dateutil import parser from django.conf import settings -import clickhouse_connect - - from rest_framework import serializers, status from rest_framework.response import Response from cvat.apps.dataset_manager.views import log_exception from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.utils import sendfile from cvat.apps.engine.rq_job_handler import RQJobMetaField +from cvat.apps.engine.utils import sendfile slogger = ServerLogManager(__name__) diff --git a/cvat/apps/events/handlers.py b/cvat/apps/events/handlers.py index 753eb84dd0da..58581dacaa37 100644 --- a/cvat/apps/events/handlers.py +++ b/cvat/apps/events/handlers.py @@ -11,26 +11,40 @@ from rest_framework.exceptions import NotAuthenticated from rest_framework.views import exception_handler -from cvat.apps.engine.models import (CloudStorage, Comment, Issue, Job, Label, - Project, ShapeType, Task, User) -from cvat.apps.engine.serializers import (BasicUserSerializer, - CloudStorageReadSerializer, - CommentReadSerializer, - IssueReadSerializer, - JobReadSerializer, LabelSerializer, - ProjectReadSerializer, - TaskReadSerializer) -from cvat.apps.organizations.models import Invitation, Membership, Organization -from cvat.apps.organizations.serializers import (InvitationReadSerializer, - MembershipReadSerializer, - OrganizationReadSerializer) +from cvat.apps.engine.models import ( + CloudStorage, + Comment, + Issue, + Job, + Label, + Project, + ShapeType, + Task, + User, +) from cvat.apps.engine.rq_job_handler import RQJobMetaField +from cvat.apps.engine.serializers import ( + BasicUserSerializer, + CloudStorageReadSerializer, + CommentReadSerializer, + IssueReadSerializer, + JobReadSerializer, + LabelSerializer, + ProjectReadSerializer, + TaskReadSerializer, +) +from cvat.apps.organizations.models import Invitation, Membership, Organization +from cvat.apps.organizations.serializers import ( + InvitationReadSerializer, + MembershipReadSerializer, + OrganizationReadSerializer, +) from cvat.apps.webhooks.models import Webhook from cvat.apps.webhooks.serializers import WebhookReadSerializer from .cache import get_cache -from .event import event_scope, record_server_event from .const import WORKING_TIME_RESOLUTION, WORKING_TIME_SCOPE +from .event import event_scope, record_server_event from .utils import compute_working_time_per_ids diff --git a/cvat/apps/events/permissions.py b/cvat/apps/events/permissions.py index a1b049cbbd4b..eb5ea2281683 100644 --- a/cvat/apps/events/permissions.py +++ b/cvat/apps/events/permissions.py @@ -4,12 +4,12 @@ # SPDX-License-Identifier: MIT from django.conf import settings - from rest_framework.exceptions import PermissionDenied from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum from cvat.utils.http import make_requests_session + class EventsPermission(OpenPolicyAgentPermission): class Scopes(StrEnum): SEND_EVENTS = 'send:events' diff --git a/cvat/apps/events/tests/test_events.py b/cvat/apps/events/tests/test_events.py index 71dcc1ea89b0..28ca3f0a20e5 100644 --- a/cvat/apps/events/tests/test_events.py +++ b/cvat/apps/events/tests/test_events.py @@ -9,10 +9,11 @@ from django.contrib.auth import get_user_model from django.test import RequestFactory -from cvat.apps.events.serializers import ClientEventsSerializer -from cvat.apps.organizations.models import Organization from cvat.apps.events.const import MAX_EVENT_DURATION, WORKING_TIME_RESOLUTION +from cvat.apps.events.serializers import ClientEventsSerializer from cvat.apps.events.utils import compute_working_time_per_ids, is_contained +from cvat.apps.organizations.models import Organization + class WorkingTimeTestCase(unittest.TestCase): _START_TIMESTAMP = datetime(2024, 1, 1, 12) diff --git a/cvat/apps/events/utils.py b/cvat/apps/events/utils.py index 745bb8fde316..2a46c48beb38 100644 --- a/cvat/apps/events/utils.py +++ b/cvat/apps/events/utils.py @@ -4,13 +4,12 @@ import datetime - -from .const import MAX_EVENT_DURATION, COMPRESSED_EVENT_SCOPES from .cache import clear_cache +from .const import COMPRESSED_EVENT_SCOPES, MAX_EVENT_DURATION def _prepare_objects_to_delete(object_to_delete): - from cvat.apps.engine.models import Project, Task, Segment, Job, Issue, Comment + from cvat.apps.engine.models import Comment, Issue, Job, Project, Segment, Task relation_chain = (Project, Task, Segment, Job, Issue, Comment) related_field_names = ('task_set', 'segment_set', 'job_set', 'issues', 'comments') diff --git a/cvat/apps/events/views.py b/cvat/apps/events/views.py index 31914a829c3b..ea1f967f81ed 100644 --- a/cvat/apps/events/views.py +++ b/cvat/apps/events/views.py @@ -4,8 +4,7 @@ from django.conf import settings from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import (OpenApiParameter, OpenApiResponse, - extend_schema) +from drf_spectacular.utils import OpenApiParameter, OpenApiResponse, extend_schema from rest_framework import status, viewsets from rest_framework.renderers import JSONRenderer from rest_framework.response import Response diff --git a/cvat/apps/health/apps.py b/cvat/apps/health/apps.py index a457048b87c9..e81a24ae6374 100644 --- a/cvat/apps/health/apps.py +++ b/cvat/apps/health/apps.py @@ -3,9 +3,9 @@ # SPDX-License-Identifier: MIT from django.apps import AppConfig - from health_check.plugins import plugin_dir + class HealthConfig(AppConfig): name = 'cvat.apps.health' diff --git a/cvat/apps/health/backends.py b/cvat/apps/health/backends.py index 2f361117173a..48484c175df0 100644 --- a/cvat/apps/health/backends.py +++ b/cvat/apps/health/backends.py @@ -3,14 +3,13 @@ # SPDX-License-Identifier: MIT import requests - +from django.conf import settings from health_check.backends import BaseHealthCheckBackend from health_check.exceptions import HealthCheckException -from django.conf import settings - from cvat.utils.http import make_requests_session + class OPAHealthCheck(BaseHealthCheckBackend): critical_service = True diff --git a/cvat/apps/health/management/commands/workerprobe.py b/cvat/apps/health/management/commands/workerprobe.py index fc8b6cf7077a..4cf6bb58d9f2 100644 --- a/cvat/apps/health/management/commands/workerprobe.py +++ b/cvat/apps/health/management/commands/workerprobe.py @@ -1,10 +1,11 @@ import os import platform from datetime import datetime, timedelta -from django.core.management.base import BaseCommand, CommandError + +import django_rq from django.conf import settings +from django.core.management.base import BaseCommand, CommandError from rq.worker import Worker -import django_rq class Command(BaseCommand): diff --git a/cvat/apps/iam/adapters.py b/cvat/apps/iam/adapters.py index 703bec48743f..50ff2812c3a5 100644 --- a/cvat/apps/iam/adapters.py +++ b/cvat/apps/iam/adapters.py @@ -2,10 +2,10 @@ # # SPDX-License-Identifier: MIT -from django.http import HttpResponseRedirect +from allauth.account.adapter import DefaultAccountAdapter from django.conf import settings +from django.http import HttpResponseRedirect -from allauth.account.adapter import DefaultAccountAdapter class DefaultAccountAdapterEx(DefaultAccountAdapter): def respond_email_verification_sent(self, request, user): diff --git a/cvat/apps/iam/admin.py b/cvat/apps/iam/admin.py index 648e15dc2da4..849b76077d14 100644 --- a/cvat/apps/iam/admin.py +++ b/cvat/apps/iam/admin.py @@ -4,8 +4,8 @@ # SPDX-License-Identifier: MIT from django.contrib import admin -from django.contrib.auth.models import Group, User from django.contrib.auth.admin import GroupAdmin, UserAdmin +from django.contrib.auth.models import Group, User from django.utils.translation import gettext_lazy as _ from cvat.apps.engine.models import Profile diff --git a/cvat/apps/iam/apps.py b/cvat/apps/iam/apps.py index 97bdc3ca05fd..e89e959256ee 100644 --- a/cvat/apps/iam/apps.py +++ b/cvat/apps/iam/apps.py @@ -5,6 +5,7 @@ from django.apps import AppConfig + class IAMConfig(AppConfig): name = 'cvat.apps.iam' diff --git a/cvat/apps/iam/authentication.py b/cvat/apps/iam/authentication.py index 412806380389..79ee8e0671b0 100644 --- a/cvat/apps/iam/authentication.py +++ b/cvat/apps/iam/authentication.py @@ -2,12 +2,14 @@ # # SPDX-License-Identifier: MIT +import hashlib + +from django.contrib.auth import get_user_model from django.core import signing +from furl import furl from rest_framework import exceptions from rest_framework.authentication import BaseAuthentication -from django.contrib.auth import get_user_model -from furl import furl -import hashlib + # Got implementation ideas in https://github.com/marcgibbons/drf_signed_auth class Signer: diff --git a/cvat/apps/iam/filters.py b/cvat/apps/iam/filters.py index 6fd62d8d05e5..73a6268ba426 100644 --- a/cvat/apps/iam/filters.py +++ b/cvat/apps/iam/filters.py @@ -2,11 +2,11 @@ # # SPDX-License-Identifier: MIT -from rest_framework.filters import BaseFilterBackend -from django.db.models import Q from collections.abc import Iterable +from django.db.models import Q from drf_spectacular.utils import OpenApiParameter +from rest_framework.filters import BaseFilterBackend ORGANIZATION_OPEN_API_PARAMETERS = [ OpenApiParameter( diff --git a/cvat/apps/iam/forms.py b/cvat/apps/iam/forms.py index c1668b924387..d46f35cc4e9e 100644 --- a/cvat/apps/iam/forms.py +++ b/cvat/apps/iam/forms.py @@ -2,13 +2,12 @@ # # SPDX-License-Identifier: MIT -from django.contrib.sites.shortcuts import get_current_site -from django.contrib.auth import get_user_model - +from allauth.account.adapter import get_adapter from allauth.account.forms import default_token_generator from allauth.account.utils import user_pk_to_url_str -from allauth.account.adapter import get_adapter from dj_rest_auth.forms import AllAuthPasswordResetForm +from django.contrib.auth import get_user_model +from django.contrib.sites.shortcuts import get_current_site UserModel = get_user_model() diff --git a/cvat/apps/iam/middleware.py b/cvat/apps/iam/middleware.py index f2f1a4bae2e0..d96a8e364086 100644 --- a/cvat/apps/iam/middleware.py +++ b/cvat/apps/iam/middleware.py @@ -5,10 +5,10 @@ from datetime import timedelta from typing import Callable -from django.utils.functional import SimpleLazyObject -from rest_framework.exceptions import ValidationError, NotFound from django.conf import settings from django.http import HttpRequest, HttpResponse +from django.utils.functional import SimpleLazyObject +from rest_framework.exceptions import NotFound, ValidationError def get_organization(request): diff --git a/cvat/apps/iam/migrations/0001_remove_business_group.py b/cvat/apps/iam/migrations/0001_remove_business_group.py index 2bf1a56b4065..a37ca5a9e3c9 100644 --- a/cvat/apps/iam/migrations/0001_remove_business_group.py +++ b/cvat/apps/iam/migrations/0001_remove_business_group.py @@ -2,7 +2,6 @@ from django.conf import settings from django.db import migrations - BUSINESS_GROUP_NAME = "business" USER_GROUP_NAME = "user" diff --git a/cvat/apps/iam/rules/tests/generate_tests.py b/cvat/apps/iam/rules/tests/generate_tests.py index 729de6732eb2..0bc788300982 100755 --- a/cvat/apps/iam/rules/tests/generate_tests.py +++ b/cvat/apps/iam/rules/tests/generate_tests.py @@ -10,8 +10,8 @@ from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor from functools import partial -from typing import Optional from pathlib import Path +from typing import Optional REPO_ROOT = Path(__file__).resolve().parents[5] diff --git a/cvat/apps/iam/schema.py b/cvat/apps/iam/schema.py index 46f9e31052c1..a09bac4817a1 100644 --- a/cvat/apps/iam/schema.py +++ b/cvat/apps/iam/schema.py @@ -9,7 +9,6 @@ from drf_spectacular.authentication import SessionScheme, TokenScheme from drf_spectacular.extensions import OpenApiAuthenticationExtension from drf_spectacular.openapi import AutoSchema - from rest_framework import serializers diff --git a/cvat/apps/iam/serializers.py b/cvat/apps/iam/serializers.py index 967b696a4f21..3ff5e01497c5 100644 --- a/cvat/apps/iam/serializers.py +++ b/cvat/apps/iam/serializers.py @@ -3,23 +3,21 @@ # # SPDX-License-Identifier: MIT -from dj_rest_auth.registration.serializers import RegisterSerializer -from dj_rest_auth.serializers import PasswordResetSerializer, LoginSerializer -from django.core.exceptions import ValidationError as DjangoValidationError -from rest_framework.exceptions import ValidationError -from rest_framework import serializers +from typing import Optional, Union + from allauth.account import app_settings as allauth_settings -from allauth.account.utils import filter_users_by_email from allauth.account.adapter import get_adapter -from allauth.account.utils import setup_user_email from allauth.account.models import EmailAddress - +from allauth.account.utils import filter_users_by_email, setup_user_email +from dj_rest_auth.registration.serializers import RegisterSerializer +from dj_rest_auth.serializers import LoginSerializer, PasswordResetSerializer from django.conf import settings from django.contrib.auth import get_user_model from django.contrib.auth.models import User - +from django.core.exceptions import ValidationError as DjangoValidationError from drf_spectacular.utils import extend_schema_field -from typing import Optional, Union +from rest_framework import serializers +from rest_framework.exceptions import ValidationError from cvat.apps.iam.forms import ResetPasswordFormEx from cvat.apps.iam.utils import get_dummy_user diff --git a/cvat/apps/iam/signals.py b/cvat/apps/iam/signals.py index 73f919a1a4a4..bce73de3dd92 100644 --- a/cvat/apps/iam/signals.py +++ b/cvat/apps/iam/signals.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: MIT from django.conf import settings -from django.contrib.auth.models import User, Group -from django.db.models.signals import post_save, post_migrate +from django.contrib.auth.models import Group, User +from django.db.models.signals import post_migrate, post_save def register_groups(sender, **kwargs): @@ -61,6 +61,7 @@ def register_signals(app_config): post_save.connect(create_user, sender=User) elif settings.IAM_TYPE == 'LDAP': import django_auth_ldap.backend + # Map groups from LDAP to roles, convert a user to super user if he/she # has an admin group. django_auth_ldap.backend.populate_user.connect(create_user) diff --git a/cvat/apps/iam/tests/test_rest_api.py b/cvat/apps/iam/tests/test_rest_api.py index d3de9fd6f1df..764e7eec87eb 100644 --- a/cvat/apps/iam/tests/test_rest_api.py +++ b/cvat/apps/iam/tests/test_rest_api.py @@ -3,18 +3,16 @@ # # SPDX-License-Identifier: MIT -from django.urls import reverse +from allauth.account.views import EmailVerificationSentView +from django.test import override_settings +from django.urls import path, re_path, reverse from rest_framework import status -from rest_framework.test import APITestCase, APIClient from rest_framework.authtoken.models import Token -from django.test import override_settings -from django.urls import path, re_path -from allauth.account.views import EmailVerificationSentView +from rest_framework.test import APIClient, APITestCase from cvat.apps.iam.urls import urlpatterns as iam_url_patterns from cvat.apps.iam.views import ConfirmEmailViewEx - urlpatterns = iam_url_patterns + [ re_path(r'^account-confirm-email/(?P[-:\w]+)/$', ConfirmEmailViewEx.as_view(), name='account_confirm_email'), diff --git a/cvat/apps/iam/urls.py b/cvat/apps/iam/urls.py index 8b8135fc2d9a..c0b1dcbe7d86 100644 --- a/cvat/apps/iam/urls.py +++ b/cvat/apps/iam/urls.py @@ -3,17 +3,23 @@ # # SPDX-License-Identifier: MIT -from django.urls import path, re_path +from allauth.account import app_settings as allauth_settings +from dj_rest_auth.views import ( + LogoutView, + PasswordChangeView, + PasswordResetConfirmView, + PasswordResetView, +) from django.conf import settings +from django.urls import path, re_path from django.urls.conf import include -from dj_rest_auth.views import ( - LogoutView, PasswordChangeView, - PasswordResetView, PasswordResetConfirmView) -from allauth.account import app_settings as allauth_settings from cvat.apps.iam.views import ( - SigningView, RegisterViewEx, RulesView, - ConfirmEmailViewEx, LoginViewEx + ConfirmEmailViewEx, + LoginViewEx, + RegisterViewEx, + RulesView, + SigningView, ) BASIC_LOGIN_PATH_NAME = 'rest_login' diff --git a/cvat/apps/iam/utils.py b/cvat/apps/iam/utils.py index 8095902769f3..5f5bee19352f 100644 --- a/cvat/apps/iam/utils.py +++ b/cvat/apps/iam/utils.py @@ -1,9 +1,9 @@ -from pathlib import Path import functools import hashlib import importlib import io import tarfile +from pathlib import Path from django.conf import settings from django.contrib.sessions.backends.base import SessionBase @@ -30,8 +30,8 @@ def add_opa_rules_path(path: Path) -> None: get_opa_bundle.cache_clear() def get_dummy_user(email): - from allauth.account.models import EmailAddress from allauth.account import app_settings + from allauth.account.models import EmailAddress from allauth.account.utils import filter_users_by_email users = filter_users_by_email(email) diff --git a/cvat/apps/iam/views.py b/cvat/apps/iam/views.py index 928d170c3bc4..b17be7ac7cb3 100644 --- a/cvat/apps/iam/views.py +++ b/cvat/apps/iam/views.py @@ -5,31 +5,34 @@ import functools -from django.http import Http404, HttpResponseBadRequest, HttpResponseRedirect -from rest_framework import views, serializers -from rest_framework.exceptions import ValidationError -from rest_framework.permissions import AllowAny -from django.conf import settings -from django.http import HttpResponse -from django.views.decorators.http import etag as django_etag -from rest_framework.response import Response +from allauth.account import app_settings as allauth_settings +from allauth.account.utils import complete_signup, has_verified_email, send_email_confirmation +from allauth.account.views import ConfirmEmailView from dj_rest_auth.app_settings import api_settings as dj_rest_auth_settings from dj_rest_auth.registration.views import RegisterView from dj_rest_auth.utils import jwt_encode from dj_rest_auth.views import LoginView -from allauth.account import app_settings as allauth_settings -from allauth.account.views import ConfirmEmailView -from allauth.account.utils import complete_signup, has_verified_email, send_email_confirmation - -from furl import furl - -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import OpenApiResponse, extend_schema, inline_serializer, extend_schema_view +from django.conf import settings +from django.http import Http404, HttpResponse, HttpResponseBadRequest, HttpResponseRedirect +from django.views.decorators.http import etag as django_etag from drf_spectacular.contrib.rest_auth import get_token_serializer_class +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiResponse, + extend_schema, + extend_schema_view, + inline_serializer, +) +from furl import furl +from rest_framework import serializers, views +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import AllowAny +from rest_framework.response import Response from .authentication import Signer from .utils import get_opa_bundle + @extend_schema(tags=['auth']) @extend_schema_view(post=extend_schema( summary='This method signs URL for access to the server', diff --git a/cvat/apps/lambda_manager/models.py b/cvat/apps/lambda_manager/models.py index 47d732c41dd1..f6e684a1cc0f 100644 --- a/cvat/apps/lambda_manager/models.py +++ b/cvat/apps/lambda_manager/models.py @@ -5,6 +5,7 @@ import django.db.models as models + class FunctionKind(models.TextChoices): DETECTOR = "detector" INTERACTOR = "interactor" diff --git a/cvat/apps/lambda_manager/permissions.py b/cvat/apps/lambda_manager/permissions.py index 94800f0edd5d..06a43253fa3c 100644 --- a/cvat/apps/lambda_manager/permissions.py +++ b/cvat/apps/lambda_manager/permissions.py @@ -8,6 +8,7 @@ from cvat.apps.engine.permissions import JobPermission, TaskPermission from cvat.apps.iam.permissions import OpenPolicyAgentPermission, StrEnum + class LambdaPermission(OpenPolicyAgentPermission): class Scopes(StrEnum): LIST = 'list' diff --git a/cvat/apps/lambda_manager/serializers.py b/cvat/apps/lambda_manager/serializers.py index ab8809bd7cc8..96ffffa5702c 100644 --- a/cvat/apps/lambda_manager/serializers.py +++ b/cvat/apps/lambda_manager/serializers.py @@ -5,6 +5,7 @@ from drf_spectacular.utils import extend_schema_serializer from rest_framework import serializers + class SublabelMappingEntrySerializer(serializers.Serializer): name = serializers.CharField() attributes = serializers.DictField(child=serializers.CharField(), required=False) diff --git a/cvat/apps/lambda_manager/tests/test_lambda.py b/cvat/apps/lambda_manager/tests/test_lambda.py index f9292b278b45..3e5cc43f8743 100644 --- a/cvat/apps/lambda_manager/tests/test_lambda.py +++ b/cvat/apps/lambda_manager/tests/test_lambda.py @@ -3,12 +3,12 @@ # # SPDX-License-Identifier: MIT +import json +import os from collections import Counter, OrderedDict from itertools import groupby from typing import Optional from unittest import mock, skip -import json -import os import requests from django.contrib.auth.models import Group, User @@ -16,7 +16,11 @@ from rest_framework import status from cvat.apps.engine.tests.utils import ( - ApiTestBase, filter_dict, ForceLogin, generate_image_file, get_paginated_collection + ApiTestBase, + ForceLogin, + filter_dict, + generate_image_file, + get_paginated_collection, ) LAMBDA_ROOT_PATH = '/api/lambda' diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 559ef29813b5..3df3476bf083 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -19,34 +19,45 @@ import numpy as np import requests import rq -from cvat.apps.events.handlers import handle_function_call -from cvat.apps.lambda_manager.signals import interactive_function_call_signal from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import (OpenApiParameter, OpenApiResponse, - extend_schema, extend_schema_view, - inline_serializer) +from drf_spectacular.utils import ( + OpenApiParameter, + OpenApiResponse, + extend_schema, + extend_schema_view, + inline_serializer, +) from rest_framework import serializers, status, viewsets -from rest_framework.response import Response from rest_framework.request import Request +from rest_framework.response import Response import cvat.apps.dataset_manager as dm from cvat.apps.engine.frame_provider import TaskFrameProvider +from cvat.apps.engine.log import ServerLogManager from cvat.apps.engine.models import ( - Job, ShapeType, SourceType, Task, Label, RequestAction, RequestTarget + Job, + Label, + RequestAction, + RequestTarget, + ShapeType, + SourceType, + Task, ) from cvat.apps.engine.rq_job_handler import RQId, RQJobMetaField from cvat.apps.engine.serializers import LabeledDataSerializer +from cvat.apps.engine.utils import define_dependent_job, get_rq_job_meta, get_rq_lock_by_user +from cvat.apps.events.handlers import handle_function_call +from cvat.apps.iam.filters import ORGANIZATION_OPEN_API_PARAMETERS from cvat.apps.lambda_manager.models import FunctionKind from cvat.apps.lambda_manager.permissions import LambdaPermission from cvat.apps.lambda_manager.serializers import ( - FunctionCallRequestSerializer, FunctionCallSerializer + FunctionCallRequestSerializer, + FunctionCallSerializer, ) -from cvat.apps.engine.log import ServerLogManager -from cvat.apps.engine.utils import define_dependent_job, get_rq_job_meta, get_rq_lock_by_user +from cvat.apps.lambda_manager.signals import interactive_function_call_signal from cvat.utils.http import make_requests_session -from cvat.apps.iam.filters import ORGANIZATION_OPEN_API_PARAMETERS slogger = ServerLogManager(__name__) diff --git a/cvat/apps/log_viewer/views.py b/cvat/apps/log_viewer/views.py index 9e52f546c634..0b20327f4658 100644 --- a/cvat/apps/log_viewer/views.py +++ b/cvat/apps/log_viewer/views.py @@ -4,11 +4,11 @@ from django.conf import settings from django.http import HttpResponsePermanentRedirect +from drf_spectacular.utils import extend_schema from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.response import Response -from drf_spectacular.utils import extend_schema @extend_schema(exclude=True) class LogViewerAccessViewSet(viewsets.ViewSet): diff --git a/cvat/apps/organizations/admin.py b/cvat/apps/organizations/admin.py index 756100244743..ca19407670f4 100644 --- a/cvat/apps/organizations/admin.py +++ b/cvat/apps/organizations/admin.py @@ -2,9 +2,11 @@ # # SPDX-License-Identifier: MIT -from .models import Organization, Membership from django.contrib import admin +from .models import Membership, Organization + + class MembershipInline(admin.TabularInline): model = Membership extra = 0 diff --git a/cvat/apps/organizations/apps.py b/cvat/apps/organizations/apps.py index f73094af1723..518a646b3e94 100644 --- a/cvat/apps/organizations/apps.py +++ b/cvat/apps/organizations/apps.py @@ -5,6 +5,7 @@ from django.apps import AppConfig + class OrganizationsConfig(AppConfig): name = 'cvat.apps.organizations' diff --git a/cvat/apps/organizations/migrations/0001_initial.py b/cvat/apps/organizations/migrations/0001_initial.py index 1d2689d343d1..cc2ecb76cf5e 100644 --- a/cvat/apps/organizations/migrations/0001_initial.py +++ b/cvat/apps/organizations/migrations/0001_initial.py @@ -1,8 +1,8 @@ # Generated by Django 3.2.8 on 2021-10-26 14:52 +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion class Migration(migrations.Migration): diff --git a/cvat/apps/organizations/models.py b/cvat/apps/organizations/models.py index 3da77bafbebf..530f79b1ada3 100644 --- a/cvat/apps/organizations/models.py +++ b/cvat/apps/organizations/models.py @@ -4,19 +4,20 @@ # SPDX-License-Identifier: MIT from datetime import timedelta -from django.conf import settings -from allauth.account.adapter import get_adapter -from django.contrib.sites.shortcuts import get_current_site -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import extend_schema_field -from django.db import models +from allauth.account.adapter import get_adapter +from django.conf import settings from django.contrib.auth import get_user_model +from django.contrib.sites.shortcuts import get_current_site from django.core.exceptions import ImproperlyConfigured +from django.db import models from django.utils import timezone +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import extend_schema_field from cvat.apps.engine.models import TimestampedModel + class Organization(TimestampedModel): slug = models.SlugField(max_length=16, blank=False, unique=True) name = models.CharField(max_length=64, blank=True) diff --git a/cvat/apps/organizations/permissions.py b/cvat/apps/organizations/permissions.py index e45b05d978c3..8eaa9f074f63 100644 --- a/cvat/apps/organizations/permissions.py +++ b/cvat/apps/organizations/permissions.py @@ -9,6 +9,7 @@ from .models import Membership + class OrganizationPermission(OpenPolicyAgentPermission): class Scopes(StrEnum): LIST = 'list' diff --git a/cvat/apps/organizations/serializers.py b/cvat/apps/organizations/serializers.py index 9cfb467aa3b9..3e6a712473f9 100644 --- a/cvat/apps/organizations/serializers.py +++ b/cvat/apps/organizations/serializers.py @@ -3,19 +3,21 @@ # # SPDX-License-Identifier: MIT -from attr.converters import to_bool -from django.contrib.auth import get_user_model from allauth.account.models import EmailAddress -from django.core.exceptions import ObjectDoesNotExist +from attr.converters import to_bool from django.conf import settings +from django.contrib.auth import get_user_model from django.contrib.auth.models import User +from django.core.exceptions import ObjectDoesNotExist from django.db import transaction - from rest_framework import serializers + from cvat.apps.engine.serializers import BasicUserSerializer from cvat.apps.iam.utils import get_dummy_user + from .models import Invitation, Membership, Organization + class OrganizationReadSerializer(serializers.ModelSerializer): owner = BasicUserSerializer(allow_null=True) class Meta: diff --git a/cvat/apps/organizations/throttle.py b/cvat/apps/organizations/throttle.py index 438538b61d4a..c99db364030c 100644 --- a/cvat/apps/organizations/throttle.py +++ b/cvat/apps/organizations/throttle.py @@ -4,5 +4,6 @@ from rest_framework.throttling import UserRateThrottle + class ResendOrganizationInvitationThrottle(UserRateThrottle): rate = '5/hour' diff --git a/cvat/apps/organizations/urls.py b/cvat/apps/organizations/urls.py index 068f72b0968d..1e68d0299cc3 100644 --- a/cvat/apps/organizations/urls.py +++ b/cvat/apps/organizations/urls.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT from rest_framework.routers import DefaultRouter + from .views import InvitationViewSet, MembershipViewSet, OrganizationViewSet router = DefaultRouter(trailing_slash=False) diff --git a/cvat/apps/organizations/views.py b/cvat/apps/organizations/views.py index 11b92b29cad8..f628bff3e1d9 100644 --- a/cvat/apps/organizations/views.py +++ b/cvat/apps/organizations/views.py @@ -3,30 +3,35 @@ # # SPDX-License-Identifier: MIT -from django.utils.crypto import get_random_string -from django.db import transaction from django.core.exceptions import ImproperlyConfigured - -from rest_framework import mixins, viewsets, status -from rest_framework.permissions import SAFE_METHODS +from django.db import transaction +from django.utils.crypto import get_random_string +from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view +from rest_framework import mixins, status, viewsets from rest_framework.decorators import action +from rest_framework.permissions import SAFE_METHODS from rest_framework.response import Response -from drf_spectacular.utils import OpenApiResponse, extend_schema, extend_schema_view - +from cvat.apps.engine.mixins import PartialUpdateModelMixin from cvat.apps.iam.filters import ORGANIZATION_OPEN_API_PARAMETERS from cvat.apps.organizations.permissions import ( - InvitationPermission, MembershipPermission, OrganizationPermission) + InvitationPermission, + MembershipPermission, + OrganizationPermission, +) from cvat.apps.organizations.throttle import ResendOrganizationInvitationThrottle -from cvat.apps.engine.mixins import PartialUpdateModelMixin from .models import Invitation, Membership, Organization - from .serializers import ( - InvitationReadSerializer, InvitationWriteSerializer, - MembershipReadSerializer, MembershipWriteSerializer, - OrganizationReadSerializer, OrganizationWriteSerializer, - AcceptInvitationReadSerializer) + AcceptInvitationReadSerializer, + InvitationReadSerializer, + InvitationWriteSerializer, + MembershipReadSerializer, + MembershipWriteSerializer, + OrganizationReadSerializer, + OrganizationWriteSerializer, +) + @extend_schema(tags=['organizations']) @extend_schema_view( diff --git a/cvat/apps/webhooks/apps.py b/cvat/apps/webhooks/apps.py index ac193baed755..50ba88aa6278 100644 --- a/cvat/apps/webhooks/apps.py +++ b/cvat/apps/webhooks/apps.py @@ -9,7 +9,7 @@ class WebhooksConfig(AppConfig): name = "cvat.apps.webhooks" def ready(self): - from . import signals # pylint: disable=unused-import - from cvat.apps.iam.permissions import load_app_permissions load_app_permissions(self) + + from . import signals # pylint: disable=unused-import diff --git a/cvat/apps/webhooks/migrations/0001_initial.py b/cvat/apps/webhooks/migrations/0001_initial.py index fe8f296b0514..49a8af2f582c 100644 --- a/cvat/apps/webhooks/migrations/0001_initial.py +++ b/cvat/apps/webhooks/migrations/0001_initial.py @@ -1,9 +1,10 @@ # Generated by Django 3.2.15 on 2022-09-19 08:26 -import cvat.apps.webhooks.models +import django.db.models.deletion from django.conf import settings from django.db import migrations, models -import django.db.models.deletion + +import cvat.apps.webhooks.models class Migration(migrations.Migration): diff --git a/cvat/apps/webhooks/permissions.py b/cvat/apps/webhooks/permissions.py index e5d132c55de6..7fe0904c7ed2 100644 --- a/cvat/apps/webhooks/permissions.py +++ b/cvat/apps/webhooks/permissions.py @@ -4,7 +4,6 @@ # SPDX-License-Identifier: MIT from django.conf import settings - from rest_framework.exceptions import ValidationError from cvat.apps.engine.models import Project @@ -13,6 +12,7 @@ from .models import WebhookTypeChoice + class WebhookPermission(OpenPolicyAgentPermission): class Scopes(StrEnum): CREATE = 'create' diff --git a/cvat/apps/webhooks/serializers.py b/cvat/apps/webhooks/serializers.py index d2bb1f309105..b3060986eefd 100644 --- a/cvat/apps/webhooks/serializers.py +++ b/cvat/apps/webhooks/serializers.py @@ -7,13 +7,8 @@ from cvat.apps.engine.models import Project from cvat.apps.engine.serializers import BasicUserSerializer, WriteOnceMixin -from .event_type import EventTypeChoice, ProjectEvents, OrganizationEvents -from .models import ( - Webhook, - WebhookContentTypeChoice, - WebhookTypeChoice, - WebhookDelivery, -) +from .event_type import EventTypeChoice, OrganizationEvents, ProjectEvents +from .models import Webhook, WebhookContentTypeChoice, WebhookDelivery, WebhookTypeChoice class EventTypeValidator: diff --git a/cvat/apps/webhooks/signals.py b/cvat/apps/webhooks/signals.py index 3e17e8f3d8f6..eb14edcdb2c3 100644 --- a/cvat/apps/webhooks/signals.py +++ b/cvat/apps/webhooks/signals.py @@ -13,17 +13,21 @@ from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.db import transaction -from django.db.models.signals import (post_delete, post_save, pre_delete, - pre_save) +from django.db.models.signals import post_delete, post_save, pre_delete, pre_save from django.dispatch import Signal, receiver from cvat.apps.engine.models import Comment, Issue, Job, Project, Task from cvat.apps.engine.serializers import BasicUserSerializer -from cvat.apps.events.handlers import (get_request, get_serializer, get_user, - get_instance_diff, organization_id, - project_id) +from cvat.apps.events.handlers import ( + get_instance_diff, + get_request, + get_serializer, + get_user, + organization_id, + project_id, +) from cvat.apps.organizations.models import Invitation, Membership, Organization -from cvat.utils.http import make_requests_session, PROXIES_FOR_UNTRUSTED_URLS +from cvat.utils.http import PROXIES_FOR_UNTRUSTED_URLS, make_requests_session from .event_type import EventTypeChoice, event_name from .models import Webhook, WebhookDelivery, WebhookTypeChoice diff --git a/cvat/apps/webhooks/urls.py b/cvat/apps/webhooks/urls.py index c309df746f96..26f86fc2313e 100644 --- a/cvat/apps/webhooks/urls.py +++ b/cvat/apps/webhooks/urls.py @@ -3,6 +3,7 @@ # SPDX-License-Identifier: MIT from rest_framework.routers import DefaultRouter + from .views import WebhookViewSet router = DefaultRouter(trailing_slash=False) diff --git a/cvat/apps/webhooks/views.py b/cvat/apps/webhooks/views.py index 66529bc6a7bd..4c084b3f3541 100644 --- a/cvat/apps/webhooks/views.py +++ b/cvat/apps/webhooks/views.py @@ -2,9 +2,13 @@ # # SPDX-License-Identifier: MIT -from drf_spectacular.utils import (OpenApiParameter, OpenApiResponse, - OpenApiTypes, extend_schema, - extend_schema_view) +from drf_spectacular.utils import ( + OpenApiParameter, + OpenApiResponse, + OpenApiTypes, + extend_schema, + extend_schema_view, +) from rest_framework import status, viewsets from rest_framework.decorators import action from rest_framework.permissions import SAFE_METHODS @@ -16,8 +20,12 @@ from .event_type import AllEvents, OrganizationEvents, ProjectEvents from .models import Webhook, WebhookDelivery, WebhookTypeChoice from .permissions import WebhookPermission -from .serializers import (EventsSerializer, WebhookDeliveryReadSerializer, - WebhookReadSerializer, WebhookWriteSerializer) +from .serializers import ( + EventsSerializer, + WebhookDeliveryReadSerializer, + WebhookReadSerializer, + WebhookWriteSerializer, +) from .signals import signal_ping, signal_redelivery diff --git a/cvat/settings/base.py b/cvat/settings/base.py index 0f6147dc4bf0..c73cb31eafa2 100644 --- a/cvat/settings/base.py +++ b/cvat/settings/base.py @@ -19,9 +19,9 @@ import os import sys import tempfile +import urllib from datetime import timedelta from enum import Enum -import urllib from attr.converters import to_bool from corsheaders.defaults import default_headers @@ -74,7 +74,7 @@ def generate_secret_key(): try: sys.path.append(BASE_DIR) - from keys.secret_key import SECRET_KEY # pylint: disable=unused-import + from keys.secret_key import SECRET_KEY # pylint: disable=unused-import except ModuleNotFoundError: generate_secret_key() from keys.secret_key import SECRET_KEY @@ -740,6 +740,7 @@ class CVAT_QUEUES(Enum): CVAT_CONCURRENT_CHUNK_PROCESSING = int(os.getenv('CVAT_CONCURRENT_CHUNK_PROCESSING', 1)) from cvat.rq_patching import update_started_job_registry_cleanup + update_started_job_registry_cleanup() CLOUD_DATA_DOWNLOADING_MAX_THREADS_NUMBER = 4 diff --git a/cvat/settings/email_settings.py b/cvat/settings/email_settings.py index d3f9621e09d4..f83f918339de 100644 --- a/cvat/settings/email_settings.py +++ b/cvat/settings/email_settings.py @@ -5,7 +5,6 @@ from cvat.settings.production import * - # https://github.com/pennersr/django-allauth ACCOUNT_AUTHENTICATION_METHOD = 'username_email' ACCOUNT_CONFIRM_EMAIL_ON_GET = True diff --git a/cvat/settings/testing.py b/cvat/settings/testing.py index 3cd47559fbd0..e0391e4c3b40 100644 --- a/cvat/settings/testing.py +++ b/cvat/settings/testing.py @@ -2,9 +2,10 @@ # # SPDX-License-Identifier: MIT -from .development import * import tempfile +from .development import * + DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', @@ -73,11 +74,13 @@ TEST_RUNNER = "cvat.settings.testing.PatchedDiscoverRunner" from django.test.runner import DiscoverRunner + + class PatchedDiscoverRunner(DiscoverRunner): def __init__(self, *args, **kwargs): # Used fakeredis for testing (don't affect production redis) - from fakeredis import FakeRedis, FakeStrictRedis import django_rq.queues + from fakeredis import FakeRedis, FakeStrictRedis simple_redis = FakeRedis() strict_redis = FakeStrictRedis() django_rq.queues.get_redis_connection = lambda _, strict: strict_redis \ diff --git a/cvat/urls.py b/cvat/urls.py index 08257a14b811..ca62b7cb03a3 100644 --- a/cvat/urls.py +++ b/cvat/urls.py @@ -20,7 +20,7 @@ from django.apps import apps from django.contrib import admin -from django.urls import path, include +from django.urls import include, path urlpatterns = [ path("admin/", admin.site.urls), diff --git a/cvat/utils/http.py b/cvat/utils/http.py index 2cb1b7498b32..ab8771aaa2ae 100644 --- a/cvat/utils/http.py +++ b/cvat/utils/http.py @@ -2,10 +2,9 @@ # # SPDX-License-Identifier: MIT -from django.conf import settings - import requests import requests.utils +from django.conf import settings from cvat import __version__ diff --git a/dev/update_version.py b/dev/update_version.py index fbe5da9971c0..7419a581ef4c 100755 --- a/dev/update_version.py +++ b/dev/update_version.py @@ -9,7 +9,6 @@ from re import Match, Pattern from typing import Callable - SUCCESS_CHAR = "\u2714" FAIL_CHAR = "\u2716" diff --git a/pyproject.toml b/pyproject.toml index 528bdc579fcc..9447beefa868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,6 +3,14 @@ profile = "black" forced_separate = ["tests"] line_length = 100 skip_gitignore = true # align tool behavior with Black +extend_skip=[ + # Correctly ordering the imports in serverless functions would + # require a pyproject.toml in every function; don't bother with it for now. + "serverless", + # Sorting the imports in this file causes test failures; + # TODO: fix them and remove this ignore. + "cvat/apps/dataset_manager/formats/registry.py", +] [tool.black] line-length = 100 diff --git a/rqscheduler.py b/rqscheduler.py index 5ae76e64a7f0..b6cebe80f285 100644 --- a/rqscheduler.py +++ b/rqscheduler.py @@ -4,10 +4,10 @@ # implementation. This is required for correct work with CVAT queue settings and # their access options such as login and password. +from rq_scheduler.scripts import rqscheduler + # Required to initialize Django settings correctly from cvat.asgi import application # pylint: disable=unused-import -from rq_scheduler.scripts import rqscheduler - if __name__ == "__main__": rqscheduler.main() diff --git a/tests/python/pyproject.toml b/tests/python/pyproject.toml index ab4db6695977..6b5fba136a78 100644 --- a/tests/python/pyproject.toml +++ b/tests/python/pyproject.toml @@ -3,3 +3,4 @@ profile = "black" forced_separate = ["tests"] line_length = 100 skip_gitignore = true # align tool behavior with Black +known_first_party = ["shared", "rest_api", "sdk", "cli"] diff --git a/utils/dataset_manifest/__init__.py b/utils/dataset_manifest/__init__.py index 74fd25ede729..7efcfcb48406 100644 --- a/utils/dataset_manifest/__init__.py +++ b/utils/dataset_manifest/__init__.py @@ -1,4 +1,4 @@ # Copyright (C) 2021-2022 Intel Corporation # # SPDX-License-Identifier: MIT -from .core import VideoManifestManager, ImageManifestManager, is_manifest +from .core import ImageManifestManager, VideoManifestManager, is_manifest diff --git a/utils/dataset_manifest/core.py b/utils/dataset_manifest/core.py index 449e70d64098..a855d170e86b 100644 --- a/utils/dataset_manifest/core.py +++ b/utils/dataset_manifest/core.py @@ -3,25 +3,24 @@ # # SPDX-License-Identifier: MIT -from enum import Enum -from io import StringIO -import av import json import os - from abc import ABC, abstractmethod from collections.abc import Iterator from contextlib import closing +from enum import Enum +from inspect import isgenerator +from io import StringIO from itertools import islice -from PIL import Image from json.decoder import JSONDecodeError -from inspect import isgenerator +from typing import Any, Callable, Optional, Union + +import av +from PIL import Image from .errors import InvalidManifestError, InvalidVideoError -from .utils import SortingMethod, md5_hash, rotate_image, sort from .types import NamedBytesIO - -from typing import Any, Union, Optional, Callable +from .utils import SortingMethod, md5_hash, rotate_image, sort class VideoStreamReader: diff --git a/utils/dataset_manifest/create.py b/utils/dataset_manifest/create.py index 64efaed60f2d..fa31300e058a 100755 --- a/utils/dataset_manifest/create.py +++ b/utils/dataset_manifest/create.py @@ -7,13 +7,14 @@ import argparse import os -import sys import re +import sys from glob import glob from tqdm import tqdm -from utils import detect_related_images, is_image, is_video, SortingMethod +from utils import SortingMethod, detect_related_images, is_image, is_video + def get_args(): parser = argparse.ArgumentParser() @@ -98,5 +99,5 @@ def main(): if __name__ == "__main__": base_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(base_dir) - from dataset_manifest.core import VideoManifestManager, ImageManifestManager + from dataset_manifest.core import ImageManifestManager, VideoManifestManager main() diff --git a/utils/dataset_manifest/types.py b/utils/dataset_manifest/types.py index 8847eee457ba..5ddcce9ad5c9 100644 --- a/utils/dataset_manifest/types.py +++ b/utils/dataset_manifest/types.py @@ -5,6 +5,7 @@ from io import BytesIO from typing import Protocol + class Named(Protocol): filename: str diff --git a/utils/dataset_manifest/utils.py b/utils/dataset_manifest/utils.py index b4eee9686b71..9cb89ce5cd4d 100644 --- a/utils/dataset_manifest/utils.py +++ b/utils/dataset_manifest/utils.py @@ -2,15 +2,17 @@ # # SPDX-License-Identifier: MIT -import os -import re import hashlib import mimetypes +import os +import re +from enum import Enum +from random import shuffle + import cv2 as cv from av import VideoFrame -from enum import Enum from natsort import os_sorted -from random import shuffle + def rotate_image(image, angle): height, width = image.shape[:2] diff --git a/utils/dicom_converter/script.py b/utils/dicom_converter/script.py index 3fe7ef0be6dd..a201845965f3 100644 --- a/utils/dicom_converter/script.py +++ b/utils/dicom_converter/script.py @@ -3,17 +3,16 @@ # SPDX-License-Identifier: MIT -import os import argparse import logging +import os from glob import glob import numpy as np -from tqdm import tqdm from PIL import Image from pydicom import dcmread from pydicom.pixel_data_handlers.util import convert_color_space - +from tqdm import tqdm # Script configuration logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(message)s") From 30fbfb2ff7a5e523d36358d3bcbe2db8f43360ff Mon Sep 17 00:00:00 2001 From: Maria Khrustaleva Date: Mon, 6 Jan 2025 15:13:28 +0100 Subject: [PATCH 2/3] Allow empty items to be cached (#8890) ### Motivation and context Details are described [here](https://github.com/cvat-ai/cvat_enterprise/pull/252). ### How has this been tested? ### Checklist - [ ] I submit my changes into the `develop` branch - [ ] I have created a changelog fragment - [ ] I have updated the documentation accordingly - [ ] I have added tests to cover my changes - [ ] I have linked related issues (see [GitHub docs]( https://help.github.com/en/github/managing-your-work-on-github/linking-a-pull-request-to-an-issue#linking-a-pull-request-to-an-issue-using-a-keyword)) - [ ] I have increased versions of npm packages if it is necessary ([cvat-canvas](https://github.com/cvat-ai/cvat/tree/develop/cvat-canvas#versioning), [cvat-core](https://github.com/cvat-ai/cvat/tree/develop/cvat-core#versioning), [cvat-data](https://github.com/cvat-ai/cvat/tree/develop/cvat-data#versioning) and [cvat-ui](https://github.com/cvat-ai/cvat/tree/develop/cvat-ui#versioning)) ### License - [ ] I submit _my code changes_ under the same [MIT License]( https://github.com/cvat-ai/cvat/blob/develop/LICENSE) that covers the project. Feel free to contact the maintainers if that's a concern. ## Summary by CodeRabbit - **Bug Fixes** - Improved cache error handling by introducing a specific exception for empty cache scenarios - Enhanced data validation when creating cache items to prevent caching of invalid or empty data --- cvat/apps/engine/cache.py | 47 +++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/cvat/apps/engine/cache.py b/cvat/apps/engine/cache.py index 43c2be7bc57e..ffe8fe0cb920 100644 --- a/cvat/apps/engine/cache.py +++ b/cvat/apps/engine/cache.py @@ -218,17 +218,19 @@ def _create_and_set_cache_item( item_data = create_callback() item_data_bytes = item_data[0].getvalue() item = (item_data[0], item_data[1], cls._get_checksum(item_data_bytes), timestamp) - if item_data_bytes: - cache = cls._cache() - with get_rq_lock_for_job( - cls._get_queue(), - key, - ): - cached_item = cache.get(key) - if cached_item is not None and timestamp <= cached_item[3]: - item = cached_item - else: - cache.set(key, item, timeout=cache_item_ttl or cache.default_timeout) + + # allow empty data to be set in cache to prevent + # future rq jobs from being enqueued to prepare the item + cache = cls._cache() + with get_rq_lock_for_job( + cls._get_queue(), + key, + ): + cached_item = cache.get(key) + if cached_item is not None and timestamp <= cached_item[3]: + item = cached_item + else: + cache.set(key, item, timeout=cache_item_ttl or cache.default_timeout) return item @@ -353,11 +355,18 @@ def _make_frame_context_images_chunk_key(self, db_data: models.Data, frame_numbe def _to_data_with_mime(self, cache_item: _CacheItem) -> DataWithMime: ... @overload - def _to_data_with_mime(self, cache_item: Optional[_CacheItem]) -> Optional[DataWithMime]: ... + def _to_data_with_mime( + self, cache_item: Optional[_CacheItem], *, allow_none: bool = False + ) -> Optional[DataWithMime]: ... - def _to_data_with_mime(self, cache_item: Optional[_CacheItem]) -> Optional[DataWithMime]: + def _to_data_with_mime( + self, cache_item: Optional[_CacheItem], *, allow_none: bool = False + ) -> Optional[DataWithMime]: if not cache_item: - return None + if allow_none: + return None + + raise ValueError("A cache item is not allowed to be None") return cache_item[:2] @@ -385,7 +394,8 @@ def get_task_chunk( return self._to_data_with_mime( self._get_cache_item( key=self._make_chunk_key(db_task, chunk_number, quality=quality), - ) + ), + allow_none=True, ) def get_or_set_task_chunk( @@ -413,7 +423,8 @@ def get_segment_task_chunk( return self._to_data_with_mime( self._get_cache_item( key=self._make_segment_task_chunk_key(db_segment, chunk_number, quality=quality), - ) + ), + allow_none=True, ) def get_or_set_segment_task_chunk( @@ -510,7 +521,9 @@ def remove_context_images_chunks(self, params: Sequence[dict[str, Any]]) -> None self._bulk_delete_cache_items(keys_to_remove) def get_cloud_preview(self, db_storage: models.CloudStorage) -> Optional[DataWithMime]: - return self._to_data_with_mime(self._get_cache_item(self._make_preview_key(db_storage))) + return self._to_data_with_mime( + self._get_cache_item(self._make_preview_key(db_storage)), allow_none=True + ) def get_or_set_cloud_preview(self, db_storage: models.CloudStorage) -> DataWithMime: return self._to_data_with_mime( From 25245e63d9bbdf0633f8b7d8259216f9f38fcaf2 Mon Sep 17 00:00:00 2001 From: Dmitrii Lavrukhin Date: Mon, 6 Jan 2025 18:29:36 +0400 Subject: [PATCH 3/3] correctly process export to yolo formats if both Train and default dataset are present (#8884) depends on https://github.com/cvat-ai/datumaro/pull/71 --- cvat/requirements/base.in | 2 +- cvat/requirements/base.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cvat/requirements/base.in b/cvat/requirements/base.in index b3900f010dda..b0d1a8f82e0b 100644 --- a/cvat/requirements/base.in +++ b/cvat/requirements/base.in @@ -12,7 +12,7 @@ azure-storage-blob==12.13.0 boto3==1.17.61 clickhouse-connect==0.6.8 coreapi==2.3.3 -datumaro @ git+https://github.com/cvat-ai/datumaro.git@232c175ef1f3b7e55bd5162353df9c86a8116fde +datumaro @ git+https://github.com/cvat-ai/datumaro.git@02507a9d41413327ca2d67a4349e5553ce763a7c dj-pagination==2.5.0 # Despite direct indication allauth in requirements we should keep 'with_social' for dj-rest-auth # to avoid possible further versions conflicts (we use registration functionality) diff --git a/cvat/requirements/base.txt b/cvat/requirements/base.txt index fce784b9481c..45db95819864 100644 --- a/cvat/requirements/base.txt +++ b/cvat/requirements/base.txt @@ -1,4 +1,4 @@ -# SHA1:5a3efd0a5c1892698d4394f019ef659275b10fdb +# SHA1:8d2c3b645ed1f9efb7ca325d5da7eb9e07aced87 # # This file is autogenerated by pip-compile-multi # To update, run: @@ -56,7 +56,7 @@ cryptography==44.0.0 # pyjwt cycler==0.12.1 # via matplotlib -datumaro @ git+https://github.com/cvat-ai/datumaro.git@232c175ef1f3b7e55bd5162353df9c86a8116fde +datumaro @ git+https://github.com/cvat-ai/datumaro.git@02507a9d41413327ca2d67a4349e5553ce763a7c # via -r cvat/requirements/base.in defusedxml==0.7.1 # via @@ -197,7 +197,7 @@ oauthlib==3.2.2 # via requests-oauthlib orderedmultidict==1.0.1 # via furl -orjson==3.10.12 +orjson==3.10.13 # via datumaro packaging==24.2 # via @@ -308,7 +308,7 @@ rq-scheduler==0.13.1 # via -r cvat/requirements/base.in rsa==4.9 # via google-auth -ruamel-yaml==0.18.6 +ruamel-yaml==0.18.7 # via datumaro ruamel-yaml-clib==0.2.12 # via ruamel-yaml