Merge remote-tracking branch 'upstream/develop' into enhance/lazy-plu…

…gin-import-v2 Signed-off-by: Kim, Vinnam <[email protected]>
openvinotoolkit · Oct 23, 2023 · dff251d · dff251d
2 parents fd2594c + c20dc3e
commit dff251d
Show file tree

Hide file tree

Showing 47 changed files with 2,206 additions and 1,397 deletions.
diff --git a/.github/workflows/nightly_check.yml b/.github/workflows/nightly_check.yml
@@ -11,14 +11,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8', '3.9', '3.10','3.11']
         include:
           - os: "macos-11"
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
           - python-version: "3.8"
             tox-env-py: "38"

diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml
@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8']
         include:
           - python-version: "3.8"
@@ -38,7 +38,7 @@ jobs:
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
     name: pr test (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/publish_sdist_to_pypi.yml b/.github/workflows/publish_sdist_to_pypi.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/weekly_check.yml b/.github/workflows/weekly_check.yml
@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8']
         include:
           - python-version: "3.8"
@@ -28,7 +28,7 @@ jobs:
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
     name: stability test (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,6 +10,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Optimize Python import to make CLI entrypoint faster
   (<https://github.com/openvinotoolkit/datumaro/pull/1182>)
 
+## 16/11/2023 - Release 1.5.1
+### Enhancements
+- Enhance Datumaro data format stream importer performance
+  (<https://github.com/openvinotoolkit/datumaro/pull/1153>)
+- Change image default dtype from float32 to uint8
+  (<https://github.com/openvinotoolkit/datumaro/pull/1175>)
+
+### Bug fixes
+- Correct explore path in the related document.
+  (<https://github.com/openvinotoolkit/datumaro/pull/1176>)
+- Fix errata in the voc document. Color values in the labelmap.txt should be separated by commas, not colons.
+  (<https://github.com/openvinotoolkit/datumaro/pull/1162>)
+- Fix hyperlink errors in the document
+  (<https://github.com/openvinotoolkit/datumaro/pull/1159>, <https://github.com/openvinotoolkit/datumaro/pull/1161>)
+- Fix memory unbounded Arrow data format export/import
+  (<https://github.com/openvinotoolkit/datumaro/pull/1169>)
+
+## 15/09/2023 - Release 1.5.0
+### New features
+- Add SAMAutomaticMaskGeneration transform
+  (<https://github.com/openvinotoolkit/datumaro/pull/1168>)
+
 ### Bug fixes
 - Coco exporter can export annotations even if there is no media, except for mask annotations which require media info.
   (<https://github.com/openvinotoolkit/datumaro/issues/1147>)(<https://github.com/openvinotoolkit/datumaro/pull/1158>)
@@ -26,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/openvinotoolkit/datumaro/pull/1129>)
 - Add SAMBboxToInstanceMask transform
   (<https://github.com/openvinotoolkit/datumaro/pull/1133>, <https://github.com/openvinotoolkit/datumaro/pull/1134>)
-- Add ConfigurableValicator
+- Add ConfigurableValidator
   (<https://github.com/openvinotoolkit/datumaro/pull/1142>)
 
 ### Enhancements

diff --git a/docs/source/docs/data-formats/formats/arrow.md b/docs/source/docs/data-formats/formats/arrow.md
@@ -178,13 +178,10 @@ Extra options for exporting to Arrow format:
   - `JPEG/95`: [JPEG](https://en.wikipedia.org/wiki/JPEG) with 95 quality
   - `JPEG/75`: [JPEG](https://en.wikipedia.org/wiki/JPEG) with 75 quality
   - `NONE`: skip saving image.
-- `--max-chunk-size MAX_CHUNK_SIZE` allow to specify maximum chunk size (batch size) when saving into arrow format.
+- `--max-shard-size MAX_SHARD_SIZE` allow to specify maximum number of dataset items when saving into arrow format.
   (default: `1000`)
 - `--num-shards NUM_SHARDS` allow to specify the number of shards to generate.
-  `--num-shards` and `--max-shard-size` are  mutually exclusive.
-  (default: `1`)
-- `--max-shard-size MAX_SHARD_SIZE` allow to specify maximum size of each shard. (e.g. 7KB = 7 \* 2^10, 3MB = 3 \* 2^20, and 2GB = 2 \* 2^30)
-  `--num-shards` and `--max-shard-size` are  mutually exclusive.
+  `--num-shards` and `--max-shard-size` are mutually exclusive.
   (default: `None`)
 - `--num-workers NUM_WORKERS` allow to multi-processing for the export. If num_workers = 0, do not use multiprocessing (default: `0`).
 

diff --git a/docs/source/docs/data-formats/formats/pascal_voc.md b/docs/source/docs/data-formats/formats/pascal_voc.md
@@ -114,7 +114,7 @@ for example:
 ```
 # label_map [label : color_rgb : parts : actions]
 helicopter:::
-elephant:0:124:134:head,ear,foot:
+elephant:0,124,134:head,ear,foot:
 ```
 It is also possible to import grayscale (1-channel) PNG masks.
 For grayscale masks provide a list of labels with the number of lines

diff --git a/docs/source/docs/jupyter_notebook_examples/transform.rst b/docs/source/docs/jupyter_notebook_examples/transform.rst
@@ -15,6 +15,7 @@ Please refer `here <https://github.com/openvinotoolkit/datumaro/blob/develop/src
    notebooks/05_transform
    notebooks/06_tiling
    notebooks/18_bbox_to_instance_mask_using_sam
+   notebooks/19_automatic_instance_mask_gen_using_sam
 
 .. grid:: 1 2 2 2
    :gutter: 2
@@ -44,6 +45,15 @@ Please refer `here <https://github.com/openvinotoolkit/datumaro/blob/develop/src
 
       This transform uses Segment Anything Model [2]_ to transform bounding box annotations to instance mask annotations.
 
+   .. grid-item-card::
+
+      .. button-ref:: notebooks/19_automatic_instance_mask_gen_using_sam
+         :color: primary
+         :outline:
+         :expand:
+
+      This transform uses Segment Anything Model [2]_ to generate instance maks annotations automatically.
+
 References
 ^^^^^^^^^^
 

diff --git a/docs/source/docs/level-up/intermediate_skills/10_data_exploration.rst b/docs/source/docs/level-up/intermediate_skills/10_data_exploration.rst
@@ -20,7 +20,7 @@ The Python example for the usage of explorer is described in :doc:`here <../../j
 
             from datumaro.components.dataset import Dataset
             from datumaro.components.environment import Environment
-            from datumaro.componenets.explorer import Explorer
+            from datumaro.components.algorithms.hash_key_inference.explorer import Explorer
 
             data_path = '/path/to/data'
 

diff --git a/docs/source/docs/release_notes.rst b/docs/source/docs/release_notes.rst
@@ -14,7 +14,7 @@ New features
 - Add multiframework (PyTorch, Tensorflow) converter
 - Add SAM OVMS and Triton server Docker image builders
 - Add SAMBboxToInstanceMask transform
-- Add ConfigurableValicator
+- Add ConfigurableValidator
 
 Enhancements
 ^^^^^^^^^^^^

diff --git a/notebooks/07_data_explorer.ipynb b/notebooks/07_data_explorer.ipynb
@@ -26,7 +26,7 @@
     "# SPDX-License-Identifier: MIT\n",
     "\n",
     "import datumaro as dm\n",
-    "from datumaro.components.explorer import Explorer\n",
+    "from datumaro.components.algorithms.hash_key_inference.explorer import Explorer\n",
     "from datumaro.components.visualizer import Visualizer"
    ]
   },
@@ -46,7 +46,7 @@
    "metadata": {},
    "source": [
     "### Explore with COCO instance segmentation dataset\n",
-    "To use data exploration, we need to define hash for each dataset. So, set `save_hash` as `True`. The default value is `False`."
+    "To use data exploration, we need to define hash for each dataset. `Explorer` calculates the hash key automatically. If you want to re-use the calculated hash key, please set `save_hashkey_meta` as `True` when exporting the dataset. The default value is `False`."
    ]
   },
   {

diff --git a/notebooks/19_automatic_instance_mask_gen_using_sam.ipynb b/notebooks/19_automatic_instance_mask_gen_using_sam.ipynb
diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -11,3 +11,4 @@ pyo3 = "0.19.2"
 serde = { version = "1.0.180", features = ["derive"] }
 serde_json = "1.0.104"
 strum = { version = "0.25", features = ["derive"] }
+tempfile = "3.8.0"