cvat/datumaro/tests/test_diff.py

from unittest import TestCase

from datumaro.components.extractor import DatasetItem, Label, Bbox
from datumaro.components.comparator import Comparator


class DiffTest(TestCase):
    def test_no_bbox_diff_with_same_item(self):
        detections = 3
        anns = [
            Bbox(i * 10, 10, 10, 10, label=i,
                    attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections)
        ]
        item = DatasetItem(id=0, annotations=anns)

        iou_thresh = 0.5
        conf_thresh = 0.5
        comp = Comparator(
            iou_threshold=iou_thresh, conf_threshold=conf_thresh)

        result = comp.compare_item_bboxes(item, item)

        matches, mispred, a_greater, b_greater = result
        self.assertEqual(0, len(mispred))
        self.assertEqual(0, len(a_greater))
        self.assertEqual(0, len(b_greater))
        self.assertEqual(len([it for it in item.annotations \
                if conf_thresh < it.attributes['score']]),
            len(matches))
        for a_bbox, b_bbox in matches:
            self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
            self.assertEqual(a_bbox.label, b_bbox.label)
            self.assertLess(conf_thresh, a_bbox.attributes['score'])
            self.assertLess(conf_thresh, b_bbox.attributes['score'])

    def test_can_find_bbox_with_wrong_label(self):
        detections = 3
        class_count = 2
        item1 = DatasetItem(id=1, annotations=[
            Bbox(i * 10, 10, 10, 10, label=i,
                    attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections)
        ])
        item2 = DatasetItem(id=2, annotations=[
            Bbox(i * 10, 10, 10, 10, label=(i + 1) % class_count,
                    attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections)
        ])

        iou_thresh = 0.5
        conf_thresh = 0.5
        comp = Comparator(
            iou_threshold=iou_thresh, conf_threshold=conf_thresh)

        result = comp.compare_item_bboxes(item1, item2)

        matches, mispred, a_greater, b_greater = result
        self.assertEqual(len([it for it in item1.annotations \
                if conf_thresh < it.attributes['score']]),
            len(mispred))
        self.assertEqual(0, len(a_greater))
        self.assertEqual(0, len(b_greater))
        self.assertEqual(0, len(matches))
        for a_bbox, b_bbox in mispred:
            self.assertLess(iou_thresh, a_bbox.iou(b_bbox))
            self.assertEqual((a_bbox.label + 1) % class_count, b_bbox.label)
            self.assertLess(conf_thresh, a_bbox.attributes['score'])
            self.assertLess(conf_thresh, b_bbox.attributes['score'])

    def test_can_find_missing_boxes(self):
        detections = 3
        class_count = 2
        item1 = DatasetItem(id=1, annotations=[
            Bbox(i * 10, 10, 10, 10, label=i,
                    attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections) if i % 2 == 0
        ])
        item2 = DatasetItem(id=2, annotations=[
            Bbox(i * 10, 10, 10, 10, label=(i + 1) % class_count,
                    attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections) if i % 2 == 1
        ])

        iou_thresh = 0.5
        conf_thresh = 0.5
        comp = Comparator(
            iou_threshold=iou_thresh, conf_threshold=conf_thresh)

        result = comp.compare_item_bboxes(item1, item2)

        matches, mispred, a_greater, b_greater = result
        self.assertEqual(0, len(mispred))
        self.assertEqual(len([it for it in item1.annotations \
                if conf_thresh < it.attributes['score']]),
            len(a_greater))
        self.assertEqual(len([it for it in item2.annotations \
                if conf_thresh < it.attributes['score']]),
            len(b_greater))
        self.assertEqual(0, len(matches))

    def test_no_label_diff_with_same_item(self):
        detections = 3
        anns = [
            Label(i, attributes={'score': (1.0 + i) / detections}) \
                for i in range(detections)
        ]
        item = DatasetItem(id=1, annotations=anns)

        conf_thresh = 0.5
        comp = Comparator(conf_threshold=conf_thresh)

        result = comp.compare_item_labels(item, item)

        matches, a_greater, b_greater = result
        self.assertEqual(0, len(a_greater))
        self.assertEqual(0, len(b_greater))
        self.assertEqual(len([it for it in item.annotations \
                if conf_thresh < it.attributes['score']]),
            len(matches))

    def test_can_find_wrong_label(self):
        item1 = DatasetItem(id=1, annotations=[
            Label(0),
            Label(1),
            Label(2),
        ])
        item2 = DatasetItem(id=2, annotations=[
            Label(2),
            Label(3),
            Label(4),
        ])

        conf_thresh = 0.5
        comp = Comparator(conf_threshold=conf_thresh)

        result = comp.compare_item_labels(item1, item2)

        matches, a_greater, b_greater = result
        self.assertEqual(2, len(a_greater))
        self.assertEqual(2, len(b_greater))
        self.assertEqual(1, len(matches))