Change match_empty_frames to empty_is_annotated (#8888)

Improves the implementation of the `match_empty_frames` quality setting
to work in cases when only GT or DS annotations are empty. This allows
to use precision as the primary metric in cases when all frames are
required to have at least 1 annotation.

The previous implementation only affected matching empty annotations.
The updated variant also counts any empty frames in `ds_count` and
`gt_count` so that mismatched empty annotations also included in
denominators of metrics.

Example: 

| annotations per frame | old accuracy | new accuracy | old precision |
new precision |
| - | - | - | - | - |
| ds: [empty, 1 valid]; gt: [empty, 1 valid] | 2/2 | 2/2 | 2/2 | 2/2 |
| ds: [1 extra, 1 valid]; gt: [empty, 1 valid] | 1/2 (only matches) |
1/3 (empty != extra) | 1/2 | 1/2 |
| ds: [empty, 1 valid]; gt: [1 miss, 1 valid] | 1/2 (only matches) | 1/3
(empty != miss) | 1/1 (only matches) | 1/2 |

So, it allowed undesirable situations in which 1 and the only correct
annotation in a job could be counted as 100% of precision. The updated
option prevents this.

changelog.d/20241229_221630_mzhiltso_empty_is_annotated.md

@@ -0,0 +1,6 @@
+### Changed
+
+- The `match_empty_frames` quality setting is changed to `empty_is_annotated`.
+  The updated option includes any empty frames in the final metrics instead of only
+  matching empty frames. This makes metrics such as Precision much more representative and useful.
+  (<https://github.com/cvat-ai/cvat/pull/8888>)

cvat-core/src/quality-settings.ts

@@ -38,7 +38,7 @@ export default class QualitySettings {
     #objectVisibilityThreshold: number;
     #panopticComparison: boolean;
     #compareAttributes: boolean;
-    #matchEmptyFrames: boolean;
+    #emptyIsAnnotated: boolean;
     #descriptions: Record<string, string>;
 
     constructor(initialData: SerializedQualitySettingsData) {
@@ -60,7 +60,7 @@ export default class QualitySettings {
         this.#objectVisibilityThreshold = initialData.object_visibility_threshold;
         this.#panopticComparison = initialData.panoptic_comparison;
         this.#compareAttributes = initialData.compare_attributes;
-        this.#matchEmptyFrames = initialData.match_empty_frames;
+        this.#emptyIsAnnotated = initialData.empty_is_annotated;
         this.#descriptions = initialData.descriptions;
     }
 
@@ -200,12 +200,12 @@ export default class QualitySettings {
         this.#maxValidationsPerJob = newVal;
     }
 
-    get matchEmptyFrames(): boolean {
-        return this.#matchEmptyFrames;
+    get emptyIsAnnotated(): boolean {
+        return this.#emptyIsAnnotated;
     }
 
-    set matchEmptyFrames(newVal: boolean) {
-        this.#matchEmptyFrames = newVal;
+    set emptyIsAnnotated(newVal: boolean) {
+        this.#emptyIsAnnotated = newVal;
     }
 
     get descriptions(): Record<string, string> {
@@ -236,7 +236,7 @@ export default class QualitySettings {
             target_metric: this.#targetMetric,
             target_metric_threshold: this.#targetMetricThreshold,
             max_validations_per_job: this.#maxValidationsPerJob,
-            match_empty_frames: this.#matchEmptyFrames,
+            empty_is_annotated: this.#emptyIsAnnotated,
         };
 
         return result;

cvat-core/src/server-response-types.ts

@@ -258,7 +258,7 @@ export interface SerializedQualitySettingsData {
     object_visibility_threshold?: number;
     panoptic_comparison?: boolean;
     compare_attributes?: boolean;
-    match_empty_frames?: boolean;
+    empty_is_annotated?: boolean;
     descriptions?: Record<string, string>;
 }
 

cvat-ui/src/components/quality-control/quality-control-page.tsx

@@ -223,7 +223,7 @@ function QualityControlPage(): JSX.Element {
                 settings.lowOverlapThreshold = values.lowOverlapThreshold / 100;
                 settings.iouThreshold = values.iouThreshold / 100;
                 settings.compareAttributes = values.compareAttributes;
-                settings.matchEmptyFrames = values.matchEmptyFrames;
+                settings.emptyIsAnnotated = values.emptyIsAnnotated;
 
                 settings.oksSigma = values.oksSigma / 100;
                 settings.pointSizeBase = values.pointSizeBase;

cvat-ui/src/components/quality-control/task-quality/quality-settings-form.tsx

@@ -34,7 +34,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
         lowOverlapThreshold: settings.lowOverlapThreshold * 100,
         iouThreshold: settings.iouThreshold * 100,
         compareAttributes: settings.compareAttributes,
-        matchEmptyFrames: settings.matchEmptyFrames,
+        emptyIsAnnotated: settings.emptyIsAnnotated,
 
         oksSigma: settings.oksSigma * 100,
         pointSizeBase: settings.pointSizeBase,
@@ -81,7 +81,7 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
             {makeTooltipFragment('Target metric', targetMetricDescription)}
             {makeTooltipFragment('Target metric threshold', settings.descriptions.targetMetricThreshold)}
             {makeTooltipFragment('Compare attributes', settings.descriptions.compareAttributes)}
-            {makeTooltipFragment('Match empty frames', settings.descriptions.matchEmptyFrames)}
+            {makeTooltipFragment('Empty frames are annotated', settings.descriptions.emptyIsAnnotated)}
         </>,
     );
 
@@ -198,12 +198,12 @@ export default function QualitySettingsForm(props: Readonly<Props>): JSX.Element
                 </Col>
                 <Col span={12}>
                     <Form.Item
-                        name='matchEmptyFrames'
+                        name='emptyIsAnnotated'
                         valuePropName='checked'
                         rules={[{ required: true }]}
                     >
                         <Checkbox>
-                            <Text className='cvat-text-color'>Match empty frames</Text>
+                            <Text className='cvat-text-color'>Empty frames are annotated</Text>
                         </Checkbox>
                     </Form.Item>
                 </Col>

cvat/apps/quality_control/migrations/0006_rename_match_empty_frames_qualitysettings_empty_is_annotated.py

@@ -0,0 +1,18 @@
+# Generated by Django 4.2.15 on 2024-12-29 19:08
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("quality_control", "0005_qualitysettings_match_empty"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="qualitysettings",
+            old_name="match_empty_frames",
+            new_name="empty_is_annotated",
+        ),
+    ]

cvat/apps/quality_control/models.py

@@ -235,7 +235,7 @@ class QualitySettings(models.Model):
 
     compare_attributes = models.BooleanField()
 
-    match_empty_frames = models.BooleanField(default=False)
+    empty_is_annotated = models.BooleanField(default=False)
 
     target_metric = models.CharField(
         max_length=32,

cvat/apps/quality_control/quality_reports.py

@@ -215,10 +215,11 @@ class ComparisonParameters(_Serializable):
     panoptic_comparison: bool = True
     "Use only the visible part of the masks and polygons in comparisons"
 
-    match_empty_frames: bool = False
+    empty_is_annotated: bool = False
     """
-    Consider unannotated (empty) frames as matching. If disabled, quality metrics, such as accuracy,
-    will be 0 if both GT and DS frames have no annotations. When enabled, they will be 1 instead.
+    Consider unannotated (empty) frames virtually annotated as "nothing".
+    If disabled, quality metrics, such as accuracy, will be 0 if both GT and DS frames
+    have no annotations. When enabled, they will be 1 instead.
     This will also add virtual annotations to empty frames in the comparison results.
     """
 
@@ -1977,15 +1978,20 @@ def _find_closest_unmatched_shape(shape: dm.Annotation):
             gt_label_idx = label_id_map[gt_ann.label] if gt_ann else self._UNMATCHED_IDX
             confusion_matrix[ds_label_idx, gt_label_idx] += 1
 
-        if self.settings.match_empty_frames and not gt_item.annotations and not ds_item.annotations:
+        if self.settings.empty_is_annotated:
             # Add virtual annotations for empty frames
-            valid_labels_count = 1
-            total_labels_count = 1
+            if not gt_item.annotations and not ds_item.annotations:
+                valid_labels_count = 1
+                total_labels_count = 1
 
-            valid_shapes_count = 1
-            total_shapes_count = 1
-            ds_shapes_count = 1
-            gt_shapes_count = 1
+                valid_shapes_count = 1
+                total_shapes_count = 1
+
+            if not ds_item.annotations:
+                ds_shapes_count = 1
+
+            if not gt_item.annotations:
+                gt_shapes_count = 1
 
         self._frame_results[frame_id] = ComparisonReportFrameSummary(
             annotations=self._generate_frame_annotations_summary(
@@ -2078,12 +2084,17 @@ def _generate_frame_annotations_summary(
     ) -> ComparisonReportAnnotationsSummary:
         summary = self._compute_annotations_summary(confusion_matrix, confusion_matrix_labels)
 
-        if self.settings.match_empty_frames and summary.total_count == 0:
+        if self.settings.empty_is_annotated:
             # Add virtual annotations for empty frames
-            summary.valid_count = 1
-            summary.total_count = 1
-            summary.ds_count = 1
-            summary.gt_count = 1
+            if not summary.total_count:
+                summary.valid_count = 1
+                summary.total_count = 1
+
+            if not summary.ds_count:
+                summary.ds_count = 1
+
+            if not summary.gt_count:
+                summary.gt_count = 1
 
         return summary
 
@@ -2108,14 +2119,26 @@ def _generate_dataset_annotations_summary(
             ),
         )
         mean_ious = []
-        empty_frame_count = 0
+        empty_gt_frames = set()
+        empty_ds_frames = set()
         confusion_matrix_labels, confusion_matrix, _ = self._make_zero_confusion_matrix()
 
-        for frame_result in frame_summaries.values():
+        for frame_id, frame_result in frame_summaries.items():
             confusion_matrix += frame_result.annotations.confusion_matrix.rows
 
-            if not np.any(frame_result.annotations.confusion_matrix.rows):
-                empty_frame_count += 1
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.triu_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_ds_frames.add(frame_id)
+
+            if self.settings.empty_is_annotated and not np.any(
+                frame_result.annotations.confusion_matrix.rows[
+                    np.tril_indices_from(frame_result.annotations.confusion_matrix.rows)
+                ]
+            ):
+                empty_gt_frames.add(frame_id)
 
             if annotation_components is None:
                 annotation_components = deepcopy(frame_result.annotation_components)
@@ -2128,13 +2151,13 @@ def _generate_dataset_annotations_summary(
             confusion_matrix, confusion_matrix_labels
         )
 
-        if self.settings.match_empty_frames and empty_frame_count:
+        if self.settings.empty_is_annotated:
             # Add virtual annotations for empty frames,
             # they are not included in the confusion matrix
-            annotation_summary.valid_count += empty_frame_count
-            annotation_summary.total_count += empty_frame_count
-            annotation_summary.ds_count += empty_frame_count
-            annotation_summary.gt_count += empty_frame_count
+            annotation_summary.valid_count += len(empty_ds_frames & empty_gt_frames)
+            annotation_summary.total_count += len(empty_ds_frames | empty_gt_frames)
+            annotation_summary.ds_count += len(empty_ds_frames)
+            annotation_summary.gt_count += len(empty_gt_frames)
 
         # Cannot be computed in accumulate()
         annotation_components.shape.mean_iou = np.mean(mean_ious)

cvat/apps/quality_control/serializers.py

@@ -92,15 +92,15 @@ class Meta:
             "object_visibility_threshold",
             "panoptic_comparison",
             "compare_attributes",
-            "match_empty_frames",
+            "empty_is_annotated",
         )
         read_only_fields = (
             "id",
             "task_id",
         )
 
         extra_kwargs = {k: {"required": False} for k in fields}
-        extra_kwargs.setdefault("match_empty_frames", {}).setdefault("default", False)
+        extra_kwargs.setdefault("empty_is_annotated", {}).setdefault("default", False)
 
         for field_name, help_text in {
             "target_metric": "The primary metric used for quality estimation",
@@ -166,9 +166,9 @@ class Meta:
                 Use only the visible part of the masks and polygons in comparisons
             """,
             "compare_attributes": "Enables or disables annotation attribute comparison",
-            "match_empty_frames": """
-                Count empty frames as matching. This affects target metrics like accuracy in cases
-                there are no annotations. If disabled, frames without annotations
+            "empty_is_annotated": """
+                Consider empty frames annotated as "empty". This affects target metrics like
+                accuracy in cases there are no annotations. If disabled, frames without annotations
                 are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
                 This will also add virtual annotations to empty frames in the comparison results.
             """,

cvat/schema.yml

@@ -9775,12 +9775,12 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
-            there are no annotations. If disabled, frames without annotations
+            Consider empty frames annotated as "empty". This affects target metrics like
+            accuracy in cases there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.
     PatchedTaskValidationLayoutWriteRequest:
@@ -10282,12 +10282,12 @@ components:
         compare_attributes:
           type: boolean
           description: Enables or disables annotation attribute comparison
-        match_empty_frames:
+        empty_is_annotated:
           type: boolean
           default: false
           description: |
-            Count empty frames as matching. This affects target metrics like accuracy in cases
-            there are no annotations. If disabled, frames without annotations
+            Consider empty frames annotated as "empty". This affects target metrics like
+            accuracy in cases there are no annotations. If disabled, frames without annotations
             are counted as not matching (accuracy is 0). If enabled, accuracy will be 1 instead.
             This will also add virtual annotations to empty frames in the comparison results.
     RegisterSerializerEx:

site/content/en/docs/manual/advanced/analytics-and-monitoring/auto-qa.md

@@ -385,7 +385,7 @@ Annotation quality settings have the following parameters:
 | - | - |
 | Min overlap threshold | Min overlap threshold used for the distinction between matched and unmatched shapes. Used to match all types of annotations. It corresponds to the Intersection over union (IoU) for spatial annotations, such as bounding boxes and masks. |
 | Low overlap threshold | Low overlap threshold used for the distinction between strong and weak matches. Only affects _Low overlap_ warnings. It's supposed that _Min similarity threshold_ <= _Low overlap threshold_. |
-| Match empty frames | Consider frames matched if there are no annotations both on GT and regular job frames |
+| Empty frames are annotated | Consider frames annotated as "empty" if there are no annotations on a frame. If a frame is empty in both GT and job annotations, it will be considered a matching annotation. |
 
 | _Point and Skeleton matching_ | |
 | - | - |

tests/python/rest_api/test_quality_control.py

@@ -1213,7 +1213,7 @@ def test_modified_task_produces_different_metrics(
             "compare_line_orientation",
             "panoptic_comparison",
             "point_size_base",
-            "match_empty_frames",
+            "empty_is_annotated",
         ],
     )
     def test_settings_affect_metrics(
@@ -1246,8 +1246,11 @@ def test_settings_affect_metrics(
             )
 
         new_report = self.create_quality_report(admin_user, task_id)
-        if parameter == "match_empty_frames":
+        if parameter == "empty_is_annotated":
             assert new_report["summary"]["valid_count"] != old_report["summary"]["valid_count"]
+            assert new_report["summary"]["total_count"] != old_report["summary"]["total_count"]
+            assert new_report["summary"]["ds_count"] != old_report["summary"]["ds_count"]
+            assert new_report["summary"]["gt_count"] != old_report["summary"]["gt_count"]
         else:
             assert (
                 new_report["summary"]["conflict_count"] != old_report["summary"]["conflict_count"]