From c5f22720cf3430f2d3b302425f9d5fa8969eb000 Mon Sep 17 00:00:00 2001 From: Maxim Zhiltsov Date: Wed, 8 Jan 2025 13:21:23 +0300 Subject: [PATCH] Change match_empty_frames to empty_is_annotated (#8888) Improves the implementation of the `match_empty_frames` quality setting to work in cases when only GT or DS annotations are empty. This allows to use precision as the primary metric in cases when all frames are required to have at least 1 annotation. The previous implementation only affected matching empty annotations. The updated variant also counts any empty frames in `ds_count` and `gt_count` so that mismatched empty annotations also included in denominators of metrics. Example: | annotations per frame | old accuracy | new accuracy | old precision | new precision | | - | - | - | - | - | | ds: [empty, 1 valid]; gt: [empty, 1 valid] | 2/2 | 2/2 | 2/2 | 2/2 | | ds: [1 extra, 1 valid]; gt: [empty, 1 valid] | 1/2 (only matches) | 1/3 (empty != extra) | 1/2 | 1/2 | | ds: [empty, 1 valid]; gt: [1 miss, 1 valid] | 1/2 (only matches) | 1/3 (empty != miss) | 1/1 (only matches) | 1/2 | So, it allowed undesirable situations in which 1 and the only correct annotation in a job could be counted as 100% of precision. The updated option prevents this. --- ...1229_221630_mzhiltso_empty_is_annotated.md | 6 ++ cvat-core/src/quality-settings.ts | 14 ++-- cvat-core/src/server-response-types.ts | 2 +- .../quality-control/quality-control-page.tsx | 2 +- .../task-quality/quality-settings-form.tsx | 8 +-- ...ames_qualitysettings_empty_is_annotated.py | 18 +++++ cvat/apps/quality_control/models.py | 2 +- cvat/apps/quality_control/quality_reports.py | 71 ++++++++++++------- cvat/apps/quality_control/serializers.py | 10 +-- cvat/schema.yml | 12 ++-- .../analytics-and-monitoring/auto-qa.md | 2 +- tests/python/rest_api/test_quality_control.py | 7 +- tests/python/shared/assets/cvat_db/data.json | 48 ++++++------- .../shared/assets/quality_settings.json | 48 ++++++------- 14 files changed, 150 insertions(+), 100 deletions(-) create mode 100644 changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md create mode 100644 cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py diff --git a/changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md b/changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md new file mode 100644 index 000000000000..63746abc86e2 --- /dev/null +++ b/changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md @@ -0,0 +1,6 @@ +### Changed + +- The `match_empty_frames` quality setting is changed to `empty_is_annotated`. + The updated option includes any empty frames in the final metrics instead of only + matching empty frames. This makes metrics such as Precision much more representative and useful. + () diff --git a/cvat-core/src/quality-settings.ts b/cvat-core/src/quality-settings.ts index 7c591e371cc4..bc553105c181 100644 --- a/cvat-core/src/quality-settings.ts +++ b/cvat-core/src/quality-settings.ts @@ -38,7 +38,7 @@ export default class QualitySettings { #objectVisibilityThreshold: number; #panopticComparison: boolean; #compareAttributes: boolean; - #matchEmptyFrames: boolean; + #emptyIsAnnotated: boolean; #descriptions: Record; constructor(initialData: SerializedQualitySettingsData) { @@ -60,7 +60,7 @@ export default class QualitySettings { this.#objectVisibilityThreshold = initialData.object_visibility_threshold; this.#panopticComparison = initialData.panoptic_comparison; this.#compareAttributes = initialData.compare_attributes; - this.#matchEmptyFrames = initialData.match_empty_frames; + this.#emptyIsAnnotated = initialData.empty_is_annotated; this.#descriptions = initialData.descriptions; } @@ -200,12 +200,12 @@ export default class QualitySettings { this.#maxValidationsPerJob = newVal; } - get matchEmptyFrames(): boolean { - return this.#matchEmptyFrames; + get emptyIsAnnotated(): boolean { + return this.#emptyIsAnnotated; } - set matchEmptyFrames(newVal: boolean) { - this.#matchEmptyFrames = newVal; + set emptyIsAnnotated(newVal: boolean) { + this.#emptyIsAnnotated = newVal; } get descriptions(): Record { @@ -236,7 +236,7 @@ export default class QualitySettings { target_metric: this.#targetMetric, target_metric_threshold: this.#targetMetricThreshold, max_validations_per_job: this.#maxValidationsPerJob, - match_empty_frames: this.#matchEmptyFrames, + empty_is_annotated: this.#emptyIsAnnotated, }; return result; diff --git a/cvat-core/src/server-response-types.ts b/cvat-core/src/server-response-types.ts index ea97c0730aaa..ef635d12004e 100644 --- a/cvat-core/src/server-response-types.ts +++ b/cvat-core/src/server-response-types.ts @@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData { object_visibility_threshold?: number; panoptic_comparison?: boolean; compare_attributes?: boolean; - match_empty_frames?: boolean; + empty_is_annotated?: boolean; descriptions?: Record; } diff --git a/cvat-ui/src/components/quality-control/quality-control-page.tsx b/cvat-ui/src/components/quality-control/quality-control-page.tsx index cbaa26a8dd09..afa166f6f5fa 100644 --- a/cvat-ui/src/components/quality-control/quality-control-page.tsx +++ b/cvat-ui/src/components/quality-control/quality-control-page.tsx @@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element { settings.lowOverlapThreshold = values.lowOverlapThreshold / 100; settings.iouThreshold = values.iouThreshold / 100; settings.compareAttributes = values.compareAttributes; - settings.matchEmptyFrames = values.matchEmptyFrames; + settings.emptyIsAnnotated = values.emptyIsAnnotated; settings.oksSigma = values.oksSigma / 100; settings.pointSizeBase = values.pointSizeBase; diff --git a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx index 87a727f9772b..b5218475b418 100644 --- a/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx +++ b/cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx @@ -34,7 +34,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element lowOverlapThreshold: settings.lowOverlapThreshold * 100, iouThreshold: settings.iouThreshold * 100, compareAttributes: settings.compareAttributes, - matchEmptyFrames: settings.matchEmptyFrames, + emptyIsAnnotated: settings.emptyIsAnnotated, oksSigma: settings.oksSigma * 100, pointSizeBase: settings.pointSizeBase, @@ -81,7 +81,7 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element {makeTooltipFragment('Target metric', targetMetricDescription)} {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)} {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)} - {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)} + {makeTooltipFragment('Empty frames are annotated', settings.descriptions.emptyIsAnnotated)} , ); @@ -198,12 +198,12 @@ export default function QualitySettingsForm(props: Readonly): JSX.Element - Match empty frames + Empty frames are annotated diff --git a/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py b/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py new file mode 100644 index 000000000000..ea2f74927309 --- /dev/null +++ b/cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.15 on 2024-12-29 19:08 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("quality_control", "0005_qualitysettings_match_empty"), + ] + + operations = [ + migrations.RenameField( + model_name="qualitysettings", + old_name="match_empty_frames", + new_name="empty_is_annotated", + ), + ] diff --git a/cvat/apps/quality_control/models.py b/cvat/apps/quality_control/models.py index a5359e4fe944..c521ac276f31 100644 --- a/cvat/apps/quality_control/models.py +++ b/cvat/apps/quality_control/models.py @@ -235,7 +235,7 @@ class QualitySettings(models.Model): compare_attributes = models.BooleanField() - match_empty_frames = models.BooleanField(default=False) + empty_is_annotated = models.BooleanField(default=False) target_metric = models.CharField( max_length=32, diff --git a/cvat/apps/quality_control/quality_reports.py b/cvat/apps/quality_control/quality_reports.py index 25b5c962dc26..f757aeabc61a 100644 --- a/cvat/apps/quality_control/quality_reports.py +++ b/cvat/apps/quality_control/quality_reports.py @@ -215,10 +215,11 @@ class ComparisonParameters(_Serializable): panoptic_comparison: bool = True "Use only the visible part of the masks and polygons in comparisons" - match_empty_frames: bool = False + empty_is_annotated: bool = False """ - Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy, - will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead. + Consider unannotated (empty) frames virtually annotated as "nothing". + If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames + have no annotations. When enabled, they will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. """ @@ -1977,15 +1978,20 @@ def _find_closest_unmatched_shape(shape: dm.Annotation): gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX confusion_matrix[ds_label_idx, gt_label_idx] += 1 - if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations: + if self.settings.empty_is_annotated: # Add virtual annotations for empty frames - valid_labels_count = 1 - total_labels_count = 1 + if not gt_item.annotations and not ds_item.annotations: + valid_labels_count = 1 + total_labels_count = 1 - valid_shapes_count = 1 - total_shapes_count = 1 - ds_shapes_count = 1 - gt_shapes_count = 1 + valid_shapes_count = 1 + total_shapes_count = 1 + + if not ds_item.annotations: + ds_shapes_count = 1 + + if not gt_item.annotations: + gt_shapes_count = 1 self._frame_results[frame_id] = ComparisonReportFrameSummary( annotations=self._generate_frame_annotations_summary( @@ -2078,12 +2084,17 @@ def _generate_frame_annotations_summary( ) -> ComparisonReportAnnotationsSummary: summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels) - if self.settings.match_empty_frames and summary.total_count == 0: + if self.settings.empty_is_annotated: # Add virtual annotations for empty frames - summary.valid_count = 1 - summary.total_count = 1 - summary.ds_count = 1 - summary.gt_count = 1 + if not summary.total_count: + summary.valid_count = 1 + summary.total_count = 1 + + if not summary.ds_count: + summary.ds_count = 1 + + if not summary.gt_count: + summary.gt_count = 1 return summary @@ -2108,14 +2119,26 @@ def _generate_dataset_annotations_summary( ), ) mean_ious = [] - empty_frame_count = 0 + empty_gt_frames = set() + empty_ds_frames = set() confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix() - for frame_result in frame_summaries.values(): + for frame_id, frame_result in frame_summaries.items(): confusion_matrix += frame_result.annotations.confusion_matrix.rows - if not np.any(frame_result.annotations.confusion_matrix.rows): - empty_frame_count += 1 + if self.settings.empty_is_annotated and not np.any( + frame_result.annotations.confusion_matrix.rows[ + np.triu_indices_from(frame_result.annotations.confusion_matrix.rows) + ] + ): + empty_ds_frames.add(frame_id) + + if self.settings.empty_is_annotated and not np.any( + frame_result.annotations.confusion_matrix.rows[ + np.tril_indices_from(frame_result.annotations.confusion_matrix.rows) + ] + ): + empty_gt_frames.add(frame_id) if annotation_components is None: annotation_components = deepcopy(frame_result.annotation_components) @@ -2128,13 +2151,13 @@ def _generate_dataset_annotations_summary( confusion_matrix, confusion_matrix_labels ) - if self.settings.match_empty_frames and empty_frame_count: + if self.settings.empty_is_annotated: # Add virtual annotations for empty frames, # they are not included in the confusion matrix - annotation_summary.valid_count += empty_frame_count - annotation_summary.total_count += empty_frame_count - annotation_summary.ds_count += empty_frame_count - annotation_summary.gt_count += empty_frame_count + annotation_summary.valid_count += len(empty_ds_frames & empty_gt_frames) + annotation_summary.total_count += len(empty_ds_frames | empty_gt_frames) + annotation_summary.ds_count += len(empty_ds_frames) + annotation_summary.gt_count += len(empty_gt_frames) # Cannot be computed in accumulate() annotation_components.shape.mean_iou = np.mean(mean_ious) diff --git a/cvat/apps/quality_control/serializers.py b/cvat/apps/quality_control/serializers.py index 6164abc12200..11a5e0d8b02e 100644 --- a/cvat/apps/quality_control/serializers.py +++ b/cvat/apps/quality_control/serializers.py @@ -92,7 +92,7 @@ class Meta: "object_visibility_threshold", "panoptic_comparison", "compare_attributes", - "match_empty_frames", + "empty_is_annotated", ) read_only_fields = ( "id", @@ -100,7 +100,7 @@ class Meta: ) extra_kwargs = {k: {"required": False} for k in fields} - extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False) + extra_kwargs.setdefault("empty_is_annotated", {}).setdefault("default", False) for field_name, help_text in { "target_metric": "The primary metric used for quality estimation", @@ -166,9 +166,9 @@ class Meta: Use only the visible part of the masks and polygons in comparisons """, "compare_attributes": "Enables or disables annotation attribute comparison", - "match_empty_frames": """ - Count empty frames as matching. This affects target metrics like accuracy in cases - there are no annotations. If disabled, frames without annotations + "empty_is_annotated": """ + Consider empty frames annotated as "empty". This affects target metrics like + accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. """, diff --git a/cvat/schema.yml b/cvat/schema.yml index 8af068ecc8b2..3da1b323a14b 100644 --- a/cvat/schema.yml +++ b/cvat/schema.yml @@ -9775,12 +9775,12 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty_frames: + empty_is_annotated: type: boolean default: false description: | - Count empty frames as matching. This affects target metrics like accuracy in cases - there are no annotations. If disabled, frames without annotations + Consider empty frames annotated as "empty". This affects target metrics like + accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. PatchedTaskValidationLayoutWriteRequest: @@ -10282,12 +10282,12 @@ components: compare_attributes: type: boolean description: Enables or disables annotation attribute comparison - match_empty_frames: + empty_is_annotated: type: boolean default: false description: | - Count empty frames as matching. This affects target metrics like accuracy in cases - there are no annotations. If disabled, frames without annotations + Consider empty frames annotated as "empty". This affects target metrics like + accuracy in cases there are no annotations. If disabled, frames without annotations are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead. This will also add virtual annotations to empty frames in the comparison results. RegisterSerializerEx: diff --git a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md index 21ebd2d99087..4a098c6545fa 100644 --- a/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md +++ b/site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md @@ -385,7 +385,7 @@ Annotation quality settings have the following parameters: | - | - | | Min overlap threshold | Min overlap threshold used for the distinction between matched and unmatched shapes. Used to match all types of annotations. It corresponds to the Intersection over union (IoU) for spatial annotations, such as bounding boxes and masks. | | Low overlap threshold | Low overlap threshold used for the distinction between strong and weak matches. Only affects _Low overlap_ warnings. It's supposed that _Min similarity threshold_ <= _Low overlap threshold_. | -| Match empty frames | Consider frames matched if there are no annotations both on GT and regular job frames | +| Empty frames are annotated | Consider frames annotated as "empty" if there are no annotations on a frame. If a frame is empty in both GT and job annotations, it will be considered a matching annotation. | | _Point and Skeleton matching_ | | | - | - | diff --git a/tests/python/rest_api/test_quality_control.py b/tests/python/rest_api/test_quality_control.py index d03675c9156e..56dd24bb0abb 100644 --- a/tests/python/rest_api/test_quality_control.py +++ b/tests/python/rest_api/test_quality_control.py @@ -1213,7 +1213,7 @@ def test_modified_task_produces_different_metrics( "compare_line_orientation", "panoptic_comparison", "point_size_base", - "match_empty_frames", + "empty_is_annotated", ], ) def test_settings_affect_metrics( @@ -1246,8 +1246,11 @@ def test_settings_affect_metrics( ) new_report = self.create_quality_report(admin_user, task_id) - if parameter == "match_empty_frames": + if parameter == "empty_is_annotated": assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"] + assert new_report["summary"]["total_count"] != old_report["summary"]["total_count"] + assert new_report["summary"]["ds_count"] != old_report["summary"]["ds_count"] + assert new_report["summary"]["gt_count"] != old_report["summary"]["gt_count"] else: assert ( new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"] diff --git a/tests/python/shared/assets/cvat_db/data.json b/tests/python/shared/assets/cvat_db/data.json index 5b30d421cb5a..53863fa94fcc 100644 --- a/tests/python/shared/assets/cvat_db/data.json +++ b/tests/python/shared/assets/cvat_db/data.json @@ -18173,7 +18173,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18197,7 +18197,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18221,7 +18221,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18245,7 +18245,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18269,7 +18269,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18293,7 +18293,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18317,7 +18317,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18341,7 +18341,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18365,7 +18365,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18389,7 +18389,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18413,7 +18413,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18437,7 +18437,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18461,7 +18461,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18485,7 +18485,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18509,7 +18509,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18533,7 +18533,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18557,7 +18557,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18581,7 +18581,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18605,7 +18605,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18629,7 +18629,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18653,7 +18653,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18677,7 +18677,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18701,7 +18701,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 @@ -18725,7 +18725,7 @@ "object_visibility_threshold": 0.05, "panoptic_comparison": true, "compare_attributes": true, - "match_empty_frames": false, + "empty_is_annotated": false, "target_metric": "accuracy", "target_metric_threshold": 0.7, "max_validations_per_job": 0 diff --git a/tests/python/shared/assets/quality_settings.json b/tests/python/shared/assets/quality_settings.json index 7ddc589bc7bf..dc56352fc1ef 100644 --- a/tests/python/shared/assets/quality_settings.json +++ b/tests/python/shared/assets/quality_settings.json @@ -14,7 +14,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -35,7 +35,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -56,7 +56,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -77,7 +77,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -98,7 +98,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -119,7 +119,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -140,7 +140,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -161,7 +161,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -182,7 +182,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -203,7 +203,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -224,7 +224,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -245,7 +245,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -266,7 +266,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -287,7 +287,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -308,7 +308,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -329,7 +329,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -350,7 +350,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -371,7 +371,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -392,7 +392,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -413,7 +413,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -434,7 +434,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -455,7 +455,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -476,7 +476,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09, @@ -497,7 +497,7 @@ "line_orientation_threshold": 0.1, "line_thickness": 0.01, "low_overlap_threshold": 0.8, - "match_empty_frames": false, + "empty_is_annotated": false, "max_validations_per_job": 0, "object_visibility_threshold": 0.05, "oks_sigma": 0.09,