From 8ff9f75cf060ec5e0aa92e84b50224cba4a29653 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 12 Dec 2024 14:07:15 +0000 Subject: [PATCH 1/6] ellipse shape --- .../data/dataset/instance_segmentation.py | 47 ++++++++++++++----- src/otx/core/data/dataset/tile.py | 1 + 2 files changed, 37 insertions(+), 11 deletions(-) diff --git a/src/otx/core/data/dataset/instance_segmentation.py b/src/otx/core/data/dataset/instance_segmentation.py index 2457e12934..cde68fa59f 100644 --- a/src/otx/core/data/dataset/instance_segmentation.py +++ b/src/otx/core/data/dataset/instance_segmentation.py @@ -5,13 +5,14 @@ from __future__ import annotations +from collections import defaultdict from functools import partial from typing import Callable import numpy as np import torch +from datumaro import Bbox, Ellipse, Image, Polygon from datumaro import Dataset as DmDataset -from datumaro import Image, Polygon from torchvision import tv_tensors from otx.core.data.entity.base import ImageInfo @@ -42,23 +43,47 @@ def _get_item_impl(self, index: int) -> InstanceSegDataEntity | None: ignored_labels: list[int] = [] img_data, img_shape, _ = self._get_img_data_and_shape(img) + anno_collection: dict[str, list] = defaultdict(list) + for anno in item.annotations: + anno_collection[anno.__class__.__name__].append(anno) + gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] - for annotation in item.annotations: - if isinstance(annotation, Polygon): - bbox = np.array(annotation.get_bbox(), dtype=np.float32) + if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority + for poly in anno_collection[Polygon.__name__]: + bbox = Bbox(*poly.get_bbox()).points gt_bboxes.append(bbox) - gt_labels.append(annotation.label) + gt_labels.append(poly.label) if self.include_polygons: - gt_polygons.append(annotation) + gt_polygons.append(poly) else: - gt_masks.append(polygon_to_bitmap([annotation], *img_shape)[0]) - - # convert xywh to xyxy format - bboxes = np.array(gt_bboxes, dtype=np.float32) if gt_bboxes else np.empty((0, 4)) - bboxes[:, 2:] += bboxes[:, :2] + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + elif Bbox.__name__ in anno_collection: + bboxes = anno_collection[Bbox.__name__] + gt_bboxes = [ann.points for ann in bboxes] + gt_labels = [ann.label for ann in bboxes] + for box in bboxes: + poly = Polygon(box.as_polygon()) + if self.include_polygons: + gt_polygons.append(poly) + else: + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + elif Ellipse.__name__ in anno_collection: + for ellipse in anno_collection[Ellipse.__name__]: + bbox = Bbox(*ellipse.get_bbox()).points + gt_bboxes.append(bbox) + gt_labels.append(ellipse.label) + poly = Polygon(ellipse.as_polygon(num_points=10)) + if self.include_polygons: + gt_polygons.append(poly) + else: + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + else: + msg = "No valid annotations found in the dataset." + raise ValueError(msg) + bboxes = np.stack(gt_bboxes, dtype=np.float32, axis=0) if gt_bboxes else np.empty((0, 4)) masks = np.stack(gt_masks, axis=0) if gt_masks else np.zeros((0, *img_shape), dtype=bool) labels = np.array(gt_labels, dtype=np.int64) diff --git a/src/otx/core/data/dataset/tile.py b/src/otx/core/data/dataset/tile.py index 4187935aa4..e4d04db879 100644 --- a/src/otx/core/data/dataset/tile.py +++ b/src/otx/core/data/dataset/tile.py @@ -92,6 +92,7 @@ def __init__( ) self._tile_size = tile_size self._tile_ann_func_map[AnnotationType.polygon] = OTXTileTransform._tile_polygon + self._tile_ann_func_map[AnnotationType.ellipse] = OTXTileTransform._tile_polygon self.with_full_img = with_full_img @staticmethod From f147039b0f04ea0a64b1e49d02e437bc68838727 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 12 Dec 2024 14:21:27 +0000 Subject: [PATCH 2/6] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ba61186d51..cfbed5b2df 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -146,6 +146,8 @@ All notable changes to this project will be documented in this file. () - Disable tiling classifier toggle in configurable parameters () +- Fix Ellipse shapes for Instance Segmentation + () ## \[v2.1.0\] From 704a13cb00a9c095c3a07b5e794594d0ec91ce06 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 12 Dec 2024 18:05:22 +0000 Subject: [PATCH 3/6] update transform --- src/otx/core/data/dataset/tile.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/otx/core/data/dataset/tile.py b/src/otx/core/data/dataset/tile.py index e4d04db879..7112034f7e 100644 --- a/src/otx/core/data/dataset/tile.py +++ b/src/otx/core/data/dataset/tile.py @@ -9,6 +9,7 @@ import operator import warnings from copy import deepcopy +from functools import partial from itertools import product from typing import TYPE_CHECKING, Callable @@ -92,7 +93,7 @@ def __init__( ) self._tile_size = tile_size self._tile_ann_func_map[AnnotationType.polygon] = OTXTileTransform._tile_polygon - self._tile_ann_func_map[AnnotationType.ellipse] = OTXTileTransform._tile_polygon + self._tile_ann_func_map[AnnotationType.ellipse] = partial(OTXTileTransform._tile_polygon, num_points=10) self.with_full_img = with_full_img @staticmethod @@ -101,9 +102,10 @@ def _tile_polygon( roi_box: sg.Polygon, threshold_drop_ann: float = 0.8, *args, # noqa: ARG004 - **kwargs, # noqa: ARG004 + **kwargs, ) -> Polygon | None: - polygon = sg.Polygon(ann.get_points()) + num_points = kwargs["num_points"] if kwargs.get("num_points") else 720 + polygon = sg.Polygon(ann.get_points(num_points=num_points)) # NOTE: polygon may be invalid, e.g. self-intersecting if not roi_box.intersects(polygon) or not polygon.is_valid: @@ -128,9 +130,10 @@ def _tile_polygon( inter = _apply_offset(inter, roi_box) - return ann.wrap( + return Polygon( points=[p for xy in inter.exterior.coords for p in xy], attributes=deepcopy(ann.attributes), + label=ann.label, ) def _extract_rois(self, image: Image) -> list[BboxIntCoords]: From dea970e264a513367ce84f6ce0bc0813ab461dbc Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Thu, 12 Dec 2024 19:02:35 +0000 Subject: [PATCH 4/6] update --- src/otx/core/data/dataset/tile.py | 89 ++++++++++++++++++++++++++----- 1 file changed, 76 insertions(+), 13 deletions(-) diff --git a/src/otx/core/data/dataset/tile.py b/src/otx/core/data/dataset/tile.py index 7112034f7e..611f8062e5 100644 --- a/src/otx/core/data/dataset/tile.py +++ b/src/otx/core/data/dataset/tile.py @@ -8,15 +8,15 @@ import logging as log import operator import warnings +from collections import defaultdict from copy import deepcopy -from functools import partial from itertools import product from typing import TYPE_CHECKING, Callable import numpy as np import shapely.geometry as sg import torch -from datumaro import Bbox, DatasetItem, Image, Polygon +from datumaro import Bbox, DatasetItem, Ellipse, Image, Polygon from datumaro import Dataset as DmDataset from datumaro.components.annotation import AnnotationType from datumaro.plugins.tiling import Tile @@ -93,7 +93,7 @@ def __init__( ) self._tile_size = tile_size self._tile_ann_func_map[AnnotationType.polygon] = OTXTileTransform._tile_polygon - self._tile_ann_func_map[AnnotationType.ellipse] = partial(OTXTileTransform._tile_polygon, num_points=10) + self._tile_ann_func_map[AnnotationType.ellipse] = OTXTileTransform._tile_ellipse self.with_full_img = with_full_img @staticmethod @@ -102,10 +102,47 @@ def _tile_polygon( roi_box: sg.Polygon, threshold_drop_ann: float = 0.8, *args, # noqa: ARG004 - **kwargs, + **kwargs, # noqa: ARG004 ) -> Polygon | None: - num_points = kwargs["num_points"] if kwargs.get("num_points") else 720 - polygon = sg.Polygon(ann.get_points(num_points=num_points)) + polygon = sg.Polygon(ann.get_points()) + + # NOTE: polygon may be invalid, e.g. self-intersecting + if not roi_box.intersects(polygon) or not polygon.is_valid: + return None + + # NOTE: intersection may return a GeometryCollection or MultiPolygon + inter = polygon.intersection(roi_box) + if isinstance(inter, (sg.GeometryCollection, sg.MultiPolygon)): + shapes = [(geom, geom.area) for geom in list(inter.geoms) if geom.is_valid] + if not shapes: + return None + + inter, _ = max(shapes, key=operator.itemgetter(1)) + + if not isinstance(inter, sg.Polygon) and not inter.is_valid: + return None + + prop_area = inter.area / polygon.area + + if prop_area < threshold_drop_ann: + return None + + inter = _apply_offset(inter, roi_box) + + return ann.wrap( + points=[p for xy in inter.exterior.coords for p in xy], + attributes=deepcopy(ann.attributes), + ) + + @staticmethod + def _tile_ellipse( + ann: Ellipse, + roi_box: sg.Polygon, + threshold_drop_ann: float = 0.8, + *args, # noqa: ARG004 + **kwargs, # noqa: ARG004 + ) -> Polygon | None: + polygon = sg.Polygon(ann.get_points(num_points=10)) # NOTE: polygon may be invalid, e.g. self-intersecting if not roi_box.intersects(polygon) or not polygon.is_valid: @@ -471,25 +508,51 @@ def _get_item_impl(self, index: int) -> TileInstSegDataEntity: # type: ignore[o img = item.media_as(Image) img_data, img_shape, _ = self._get_img_data_and_shape(img) + anno_collection: dict[str, list] = defaultdict(list) + for anno in item.annotations: + anno_collection[anno.__class__.__name__].append(anno) + gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] - for annotation in item.annotations: - if isinstance(annotation, Polygon): - bbox = np.array(annotation.get_bbox(), dtype=np.float32) + if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority + for poly in anno_collection[Polygon.__name__]: + bbox = Bbox(*poly.get_bbox()).points gt_bboxes.append(bbox) - gt_labels.append(annotation.label) + gt_labels.append(poly.label) if self._dataset.include_polygons: - gt_polygons.append(annotation) + gt_polygons.append(poly) + else: + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + elif Bbox.__name__ in anno_collection: + bboxes = anno_collection[Bbox.__name__] + gt_bboxes = [ann.points for ann in bboxes] + gt_labels = [ann.label for ann in bboxes] + for box in bboxes: + poly = Polygon(box.as_polygon()) + if self._dataset.include_polygons: + gt_polygons.append(poly) else: - gt_masks.append(polygon_to_bitmap([annotation], *img_shape)[0]) + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + elif Ellipse.__name__ in anno_collection: + for ellipse in anno_collection[Ellipse.__name__]: + bbox = Bbox(*ellipse.get_bbox()).points + gt_bboxes.append(bbox) + gt_labels.append(ellipse.label) + poly = Polygon(ellipse.as_polygon(num_points=10)) + if self._dataset.include_polygons: + gt_polygons.append(poly) + else: + gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) + else: + msg = "No valid annotations found in the dataset." + raise ValueError(msg) if empty_anno := len(gt_bboxes) == 0: warnings.warn(f"Empty annotation for image {item.id}", stacklevel=2) # convert xywh to xyxy format bboxes = np.empty((0, 4), dtype=np.float32) if empty_anno else np.stack(gt_bboxes, dtype=np.float32) - bboxes[:, 2:] += bboxes[:, :2] masks = np.stack(gt_masks, axis=0) if gt_masks else np.empty((0, *img_shape), dtype=bool) labels = np.array(gt_labels, dtype=np.int64) From dab90d865f44a736d9801201968aeca582d00266 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Fri, 13 Dec 2024 09:21:36 +0000 Subject: [PATCH 5/6] Allow empty anno --- .../data/dataset/instance_segmentation.py | 7 +++++-- src/otx/core/data/dataset/tile.py | 21 ++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/otx/core/data/dataset/instance_segmentation.py b/src/otx/core/data/dataset/instance_segmentation.py index cde68fa59f..8cd8ea6c07 100644 --- a/src/otx/core/data/dataset/instance_segmentation.py +++ b/src/otx/core/data/dataset/instance_segmentation.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from collections import defaultdict from functools import partial from typing import Callable @@ -49,6 +50,9 @@ def _get_item_impl(self, index: int) -> InstanceSegDataEntity | None: gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] + # NOTE (Eugene): + # Temporary solution to handle multiple annotation types. + # Ideally, we should pre-filter annotations during initialization of the dataset. if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority for poly in anno_collection[Polygon.__name__]: bbox = Bbox(*poly.get_bbox()).points @@ -80,8 +84,7 @@ def _get_item_impl(self, index: int) -> InstanceSegDataEntity | None: else: gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) else: - msg = "No valid annotations found in the dataset." - raise ValueError(msg) + warnings.warn(f"No valid annotations found for image {item.id}!", stacklevel=2) bboxes = np.stack(gt_bboxes, dtype=np.float32, axis=0) if gt_bboxes else np.empty((0, 4)) masks = np.stack(gt_masks, axis=0) if gt_masks else np.zeros((0, *img_shape), dtype=bool) diff --git a/src/otx/core/data/dataset/tile.py b/src/otx/core/data/dataset/tile.py index 611f8062e5..502dd3530a 100644 --- a/src/otx/core/data/dataset/tile.py +++ b/src/otx/core/data/dataset/tile.py @@ -514,6 +514,9 @@ def _get_item_impl(self, index: int) -> TileInstSegDataEntity: # type: ignore[o gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] + # NOTE (Eugene): + # Temporary solution to handle multiple annotation types. + # Ideally, we should pre-filter annotations during initialization of the dataset. if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority for poly in anno_collection[Polygon.__name__]: bbox = Bbox(*poly.get_bbox()).points @@ -525,10 +528,10 @@ def _get_item_impl(self, index: int) -> TileInstSegDataEntity: # type: ignore[o else: gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) elif Bbox.__name__ in anno_collection: - bboxes = anno_collection[Bbox.__name__] - gt_bboxes = [ann.points for ann in bboxes] - gt_labels = [ann.label for ann in bboxes] - for box in bboxes: + boxes = anno_collection[Bbox.__name__] + gt_bboxes = [ann.points for ann in boxes] + gt_labels = [ann.label for ann in boxes] + for box in boxes: poly = Polygon(box.as_polygon()) if self._dataset.include_polygons: gt_polygons.append(poly) @@ -545,15 +548,9 @@ def _get_item_impl(self, index: int) -> TileInstSegDataEntity: # type: ignore[o else: gt_masks.append(polygon_to_bitmap([poly], *img_shape)[0]) else: - msg = "No valid annotations found in the dataset." - raise ValueError(msg) - - if empty_anno := len(gt_bboxes) == 0: - warnings.warn(f"Empty annotation for image {item.id}", stacklevel=2) - - # convert xywh to xyxy format - bboxes = np.empty((0, 4), dtype=np.float32) if empty_anno else np.stack(gt_bboxes, dtype=np.float32) + warnings.warn(f"No valid annotations found for image {item.id}!", stacklevel=2) + bboxes = np.stack(gt_bboxes, dtype=np.float32) if gt_bboxes else np.empty((0, 4), dtype=np.float32) masks = np.stack(gt_masks, axis=0) if gt_masks else np.empty((0, *img_shape), dtype=bool) labels = np.array(gt_labels, dtype=np.int64) From 299bca1710762db36620368db55115826a3f1747 Mon Sep 17 00:00:00 2001 From: Eugene Liu Date: Fri, 13 Dec 2024 10:14:16 +0000 Subject: [PATCH 6/6] Update todo --- src/otx/core/data/dataset/instance_segmentation.py | 2 +- src/otx/core/data/dataset/tile.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/otx/core/data/dataset/instance_segmentation.py b/src/otx/core/data/dataset/instance_segmentation.py index 8cd8ea6c07..2f3821537b 100644 --- a/src/otx/core/data/dataset/instance_segmentation.py +++ b/src/otx/core/data/dataset/instance_segmentation.py @@ -50,7 +50,7 @@ def _get_item_impl(self, index: int) -> InstanceSegDataEntity | None: gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] - # NOTE (Eugene): + # TODO(Eugene): https://jira.devtools.intel.com/browse/CVS-159363 # Temporary solution to handle multiple annotation types. # Ideally, we should pre-filter annotations during initialization of the dataset. if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority diff --git a/src/otx/core/data/dataset/tile.py b/src/otx/core/data/dataset/tile.py index 502dd3530a..b94931487d 100644 --- a/src/otx/core/data/dataset/tile.py +++ b/src/otx/core/data/dataset/tile.py @@ -514,9 +514,10 @@ def _get_item_impl(self, index: int) -> TileInstSegDataEntity: # type: ignore[o gt_bboxes, gt_labels, gt_masks, gt_polygons = [], [], [], [] - # NOTE (Eugene): + # TODO(Eugene): https://jira.devtools.intel.com/browse/CVS-159363 # Temporary solution to handle multiple annotation types. # Ideally, we should pre-filter annotations during initialization of the dataset. + if Polygon.__name__ in anno_collection: # Polygon for InstSeg has higher priority for poly in anno_collection[Polygon.__name__]: bbox = Bbox(*poly.get_bbox()).points