From 17bc9207046f8a8ae53c2a2ab3ed5956d490dc5b Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Sun, 29 Dec 2024 21:46:51 +0200 Subject: [PATCH] Change match_empty_frames to empty_is_annotated --- cvat-core/src/quality-settings.ts | 14 ++-- cvat-core/src/server-response-types.ts | 2 +- .../quality-control/quality-control-page.tsx | 2 +- .../task-quality/quality-settings-form.tsx | 8 +- ...ames_qualitysettings_empty_is_annotated.py | 18 +++++ cvat/apps/quality_control/models.py | 2 +- cvat/apps/quality_control/quality_reports.py | 73 +++++++++++++------ cvat/apps/quality_control/serializers.py | 8 +- cvat/schema.yml | 8 +- tests/python/rest_api/test_quality_control.py | 7 +- tests/python/shared/assets/cvat_db/data.json | 48 ++++++------ .../shared/assets/quality_settings.json | 48 ++++++------ 12 files changed, 142 insertions(+), 96 deletions(-) create mode 100644 cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 7c591e371cc4..bc553105c181 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -38,7 +38,7 @@ export default class QualitySettings { #objectVisibilityThreshold: number; #panopticComparison: boolean; #compareAttributes: boolean; - #matchEmptyFrames: boolean; + #emptyIsAnnotated: boolean; #descriptions: Record; constructor(initialData: SerializedQualitySettingsData) { @@ -60,7 +60,7 @@ export default class QualitySettings { this.#objectVisibilityThreshold = initialData.object_visibility_threshold; this.#panopticComparison = initialData.panoptic_comparison; this.#compareAttributes = initialData.compare_attributes; - this.#matchEmptyFrames = initialData.match_empty_frames; + this.#emptyIsAnnotated = initialData.empty_is_annotated; this.#descriptions = initialData.descriptions; } @@ -200,12 +200,12 @@ export default class QualitySettings { this.#maxValidationsPerJob = newVal; } - get matchEmptyFrames(): boolean { - return this.#matchEmptyFrames; + get emptyIsAnnotated(): boolean { + return this.#emptyIsAnnotated; } - set matchEmptyFrames(newVal: boolean) { - this.#matchEmptyFrames = newVal; + set emptyIsAnnotated(newVal: boolean) { + this.#emptyIsAnnotated = newVal; } get descriptions(): Record { @@ -236,7 +236,7 @@ export default class QualitySettings { target_metric: this.#targetMetric, target_metric_threshold: this.#targetMetricThreshold, max_validations_per_job: this.#maxValidationsPerJob, - match_empty_frames: this.#matchEmptyFrames, + empty_is_annotated: this.#emptyIsAnnotated, }; return result; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index ea97c0730aaa..ef635d12004e 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData { object_visibility_threshold?: number; panoptic_comparison?: boolean; compare_attributes?: boolean; - match_empty_frames?: boolean; + empty_is_annotated?: boolean; descriptions?: Record; } diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index cbaa26a8dd09..afa166f6f5fa 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element { settings.lowOverlapThreshold = values.lowOverlapThreshold / 100; settings.iouThreshold = values.iouThreshold / 100; settings.compareAttributes = values.compareAttributes; - settings.matchEmptyFrames = values.matchEmptyFrames; + settings.emptyIsAnnotated = values.emptyIsAnnotated; settings.oksSigma = values.oksSigma / 100; settings.pointSizeBase = values.pointSizeBase; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 87a727f9772b..b5218475b418 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -34,7 +34,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element lowOverlapThreshold: settings.lowOverlapThreshold * 100, iouThreshold: settings.iouThreshold * 100, compareAttributes: settings.compareAttributes, - matchEmptyFrames: settings.matchEmptyFrames, + emptyIsAnnotated: settings.emptyIsAnnotated, oksSigma: settings.oksSigma * 100, pointSizeBase: settings.pointSizeBase, @@ -81,7 +81,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element {makeTooltipFragment('Target metric', targetMetricDescription)} {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)} {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)} - {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)} + {makeTooltipFragment('Empty frames are annotated', settings.descriptions.emptyIsAnnotated)} , ); @@ -198,12 +198,12 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element - Match empty frames + Empty frames are annotated diff --git a/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py b/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py new file mode 100644 index 000000000000..ea2f74927309 --- /dev/null +++ b/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-12-29 19:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0005_qualitysettings_match_empty"), + ] + + operations = [ + migrations.RenameField( + model_name="qualitysettings", + old_name="match_empty_frames", + new_name="empty_is_annotated", + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index a5359e4fe944..c521ac276f31 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -235,7 +235,7 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() - match_empty_frames = models.BooleanField(default=False) + empty_is_annotated = models.BooleanField(default=False) target_metric = models.CharField( max_length=32, diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 25b5c962dc26..b37ab1437363 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -215,10 +215,11 @@ class ComparisonParameters(_Serializable): panoptic_comparison: bool = True "Use only the visible part of the masks and polygons in comparisons" - match_empty_frames: bool = False + empty_is_annotated: bool = False """ - Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, - will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. + Consider unannotated (empty) frames virtually annotated as "nothing". + If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames + have no annotations. When enabled, they will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. """ @@ -1977,15 +1978,22 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX confusion_matrix[ds_label_idx, gt_label_idx] += 1 - if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations: + if self.settings.empty_is_annotated and ( + not gt_item.annotations or not ds_item.annotations + ): # Add virtual annotations for empty frames - valid_labels_count = 1 - total_labels_count = 1 + if not gt_item.annotations and not ds_item.annotations: + valid_labels_count = 1 + total_labels_count = 1 + + valid_shapes_count = 1 + total_shapes_count = 1 - valid_shapes_count = 1 - total_shapes_count = 1 - ds_shapes_count = 1 - gt_shapes_count = 1 + if not ds_item.annotations: + ds_shapes_count = 1 + + if not gt_item.annotations: + gt_shapes_count = 1 self._frame_results[frame_id] = ComparisonReportFrameSummary( annotations=self._generate_frame_annotations_summary( @@ -2078,12 +2086,17 @@ def _generate_frame_annotations_summary( ) -> ComparisonReportAnnotationsSummary: summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels) - if self.settings.match_empty_frames and summary.total_count == 0: + if self.settings.empty_is_annotated and (not summary.ds_count or not summary.gt_count): # Add virtual annotations for empty frames - summary.valid_count = 1 - summary.total_count = 1 - summary.ds_count = 1 - summary.gt_count = 1 + if not summary.total_count: + summary.valid_count = 1 + summary.total_count = 1 + + if not summary.ds_count: + summary.ds_count = 1 + + if not summary.gt_count: + summary.gt_count = 1 return summary @@ -2108,14 +2121,26 @@ def _generate_dataset_annotations_summary( ), ) mean_ious = [] - empty_frame_count = 0 + empty_gt_frames = set() + empty_ds_frames = set() confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix() - for frame_result in frame_summaries.values(): + for frame_id, frame_result in frame_summaries.items(): confusion_matrix += frame_result.annotations.confusion_matrix.rows - if not np.any(frame_result.annotations.confusion_matrix.rows): - empty_frame_count += 1 + if self.settings.empty_is_annotated and not np.any( + frame_result.annotations.confusion_matrix.rows[ + np.triu_indices_from(frame_result.annotations.confusion_matrix.rows) + ] + ): + empty_ds_frames.add(frame_id) + + if self.settings.empty_is_annotated and not np.any( + frame_result.annotations.confusion_matrix.rows[ + np.tril_indices_from(frame_result.annotations.confusion_matrix.rows) + ] + ): + empty_gt_frames.add(frame_id) if annotation_components is None: annotation_components = deepcopy(frame_result.annotation_components) @@ -2128,13 +2153,13 @@ def _generate_dataset_annotations_summary( confusion_matrix, confusion_matrix_labels ) - if self.settings.match_empty_frames and empty_frame_count: + if self.settings.empty_is_annotated: # Add virtual annotations for empty frames, # they are not included in the confusion matrix - annotation_summary.valid_count += empty_frame_count - annotation_summary.total_count += empty_frame_count - annotation_summary.ds_count += empty_frame_count - annotation_summary.gt_count += empty_frame_count + annotation_summary.valid_count += len(empty_ds_frames & empty_gt_frames) + annotation_summary.total_count += len(empty_ds_frames | empty_gt_frames) + annotation_summary.ds_count += len(empty_ds_frames) + annotation_summary.gt_count += len(empty_gt_frames) # Cannot be computed in accumulate() annotation_components.shape.mean_iou = np.mean(mean_ious) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 6164abc12200..3d6c42e1d8fd 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -92,7 +92,7 @@ class Meta: "object_visibility_threshold", "panoptic_comparison", "compare_attributes", - "match_empty_frames", + "empty_is_annotated", ) read_only_fields = ( "id", @@ -100,7 +100,7 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} - extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False) + extra_kwargs.setdefault("empty_is_annotated", {}).setdefault("default", False) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -166,8 +166,8 @@ class Meta: Use only the visible part of the masks and polygons in comparisons """, "compare_attributes": "Enables or disables annotation attribute comparison", - "match_empty_frames": """ - Count empty frames as matching. This affects target metrics like accuracy in cases + "empty_is_annotated": """ + Count empty frames as annotated. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. diff --git a/cvat/schema.yml b/cvat/schema.yml index 8af068ecc8b2..61543a3543c6 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9775,11 +9775,11 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty_frames: + empty_is_annotated: type: boolean default: false description: | - Count empty frames as matching. This affects target metrics like accuracy in cases + Count empty frames as annotated. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. @@ -10282,11 +10282,11 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty_frames: + empty_is_annotated: type: boolean default: false description: | - Count empty frames as matching. This affects target metrics like accuracy in cases + Count empty frames as annotated. This affects target metrics like accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index d03675c9156e..8c01e1b50b4e 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1213,7 +1213,7 @@ def test_modified_task_produces_different_metrics( "compare_line_orientation", "panoptic_comparison", "point_size_base", - "match_empty_frames", + "empty_is_annotated", ], ) def test_settings_affect_metrics( @@ -1246,8 +1246,11 @@ def test_settings_affect_metrics( ) new_report = self.create_quality_report(admin_user, task_id) - if parameter == "match_empty_frames": + if parameter == "empty_is_annotated": assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"] + assert new_report["summary"]["total_count`"] != old_report["summary"]["total_count`"] + assert new_report["summary"]["ds_count"] != old_report["summary"]["ds_count"] + assert new_report["summary"]["gt_count"] != old_report["summary"]["gt_count"] else: assert ( new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 5b30d421cb5a..53863fa94fcc 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18173,7 +18173,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18197,7 +18197,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18221,7 +18221,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18245,7 +18245,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18269,7 +18269,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18293,7 +18293,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18317,7 +18317,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18341,7 +18341,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18365,7 +18365,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18389,7 +18389,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18413,7 +18413,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18437,7 +18437,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18461,7 +18461,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18485,7 +18485,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18509,7 +18509,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18533,7 +18533,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18557,7 +18557,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18581,7 +18581,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18605,7 +18605,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18629,7 +18629,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18653,7 +18653,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18677,7 +18677,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18701,7 +18701,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18725,7 +18725,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 7ddc589bc7bf..dc56352fc1ef 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -14,7 +14,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -35,7 +35,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -56,7 +56,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -77,7 +77,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -98,7 +98,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -119,7 +119,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -140,7 +140,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -161,7 +161,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -182,7 +182,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -203,7 +203,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -224,7 +224,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -245,7 +245,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -266,7 +266,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -287,7 +287,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -308,7 +308,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -329,7 +329,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -350,7 +350,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -371,7 +371,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -392,7 +392,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -413,7 +413,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -434,7 +434,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -455,7 +455,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -476,7 +476,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -497,7 +497,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09,