Mergeback 1.5.1rc2 (#1181)

openvinotoolkit · Oct 23, 2023 · c20dc3e · c20dc3e
2 parents ba6f0ed + 8faca62
commit c20dc3e
Show file tree

Hide file tree

Showing 40 changed files with 1,299 additions and 1,394 deletions.
diff --git a/.github/workflows/nightly_check.yml b/.github/workflows/nightly_check.yml
@@ -11,14 +11,14 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8', '3.9', '3.10','3.11']
         include:
           - os: "macos-11"
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
           - python-version: "3.8"
             tox-env-py: "38"

diff --git a/.github/workflows/pr_check.yml b/.github/workflows/pr_check.yml
@@ -29,7 +29,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8']
         include:
           - python-version: "3.8"
@@ -38,7 +38,7 @@ jobs:
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
     name: pr test (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}

diff --git a/.github/workflows/publish_sdist_to_pypi.yml b/.github/workflows/publish_sdist_to_pypi.yml
@@ -12,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
     steps:
       - name: Checkout
         uses: actions/checkout@v3

diff --git a/.github/workflows/weekly_check.yml b/.github/workflows/weekly_check.yml
@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['macos-11', 'ubuntu-20.04', 'windows-2019']
+        os: ['macos-11', 'ubuntu-20.04', 'windows-2022']
         python-version: ['3.8']
         include:
           - python-version: "3.8"
@@ -28,7 +28,7 @@ jobs:
             tox-env-os: "darwin"
           - os: "ubuntu-20.04"
             tox-env-os: "lin"
-          - os: "windows-2019"
+          - os: "windows-2022"
             tox-env-os: "win"
     name: stability test (${{ matrix.os }}, Python ${{ matrix.python-version }})
     runs-on: ${{ matrix.os }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -6,6 +6,24 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## \[Unreleased\]
+
+### Enhancements
+- Enhance Datumaro data format stream importer performance
+  (<https://github.com/openvinotoolkit/datumaro/pull/1153>)
+- Change image default dtype from float32 to uint8
+  (<https://github.com/openvinotoolkit/datumaro/pull/1175>)
+
+### Bug fixes
+- Correct explore path in the related document.
+  (<https://github.com/openvinotoolkit/datumaro/pull/1176>)
+- Fix errata in the voc document. Color values in the labelmap.txt should be separated by commas, not colons.
+  (<https://github.com/openvinotoolkit/datumaro/pull/1162>)
+- Fix hyperlink errors in the document
+  (<https://github.com/openvinotoolkit/datumaro/pull/1159>, <https://github.com/openvinotoolkit/datumaro/pull/1161>)
+- Fix memory unbounded Arrow data format export/import
+  (<https://github.com/openvinotoolkit/datumaro/pull/1169>)
+
+## 15/09/2023 - Release 1.5.0
 ### New features
 - Add SAMAutomaticMaskGeneration transform
   (<https://github.com/openvinotoolkit/datumaro/pull/1168>)
@@ -26,7 +44,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (<https://github.com/openvinotoolkit/datumaro/pull/1129>)
 - Add SAMBboxToInstanceMask transform
   (<https://github.com/openvinotoolkit/datumaro/pull/1133>, <https://github.com/openvinotoolkit/datumaro/pull/1134>)
-- Add ConfigurableValicator
+- Add ConfigurableValidator
   (<https://github.com/openvinotoolkit/datumaro/pull/1142>)
 
 ### Enhancements

diff --git a/docs/source/docs/data-formats/formats/arrow.md b/docs/source/docs/data-formats/formats/arrow.md
@@ -178,13 +178,10 @@ Extra options for exporting to Arrow format:
   - `JPEG/95`: [JPEG](https://en.wikipedia.org/wiki/JPEG) with 95 quality
   - `JPEG/75`: [JPEG](https://en.wikipedia.org/wiki/JPEG) with 75 quality
   - `NONE`: skip saving image.
-- `--max-chunk-size MAX_CHUNK_SIZE` allow to specify maximum chunk size (batch size) when saving into arrow format.
+- `--max-shard-size MAX_SHARD_SIZE` allow to specify maximum number of dataset items when saving into arrow format.
   (default: `1000`)
 - `--num-shards NUM_SHARDS` allow to specify the number of shards to generate.
-  `--num-shards` and `--max-shard-size` are  mutually exclusive.
-  (default: `1`)
-- `--max-shard-size MAX_SHARD_SIZE` allow to specify maximum size of each shard. (e.g. 7KB = 7 \* 2^10, 3MB = 3 \* 2^20, and 2GB = 2 \* 2^30)
-  `--num-shards` and `--max-shard-size` are  mutually exclusive.
+  `--num-shards` and `--max-shard-size` are mutually exclusive.
   (default: `None`)
 - `--num-workers NUM_WORKERS` allow to multi-processing for the export. If num_workers = 0, do not use multiprocessing (default: `0`).
 

diff --git a/docs/source/docs/data-formats/formats/pascal_voc.md b/docs/source/docs/data-formats/formats/pascal_voc.md
@@ -114,7 +114,7 @@ for example:
 ```
 # label_map [label : color_rgb : parts : actions]
 helicopter:::
-elephant:0:124:134:head,ear,foot:
+elephant:0,124,134:head,ear,foot:
 ```
 It is also possible to import grayscale (1-channel) PNG masks.
 For grayscale masks provide a list of labels with the number of lines

diff --git a/docs/source/docs/level-up/intermediate_skills/10_data_exploration.rst b/docs/source/docs/level-up/intermediate_skills/10_data_exploration.rst
@@ -20,7 +20,7 @@ The Python example for the usage of explorer is described in :doc:`here <../../j
 
             from datumaro.components.dataset import Dataset
             from datumaro.components.environment import Environment
-            from datumaro.componenets.explorer import Explorer
+            from datumaro.components.algorithms.hash_key_inference.explorer import Explorer
 
             data_path = '/path/to/data'
 

diff --git a/docs/source/docs/release_notes.rst b/docs/source/docs/release_notes.rst
@@ -14,7 +14,7 @@ New features
 - Add multiframework (PyTorch, Tensorflow) converter
 - Add SAM OVMS and Triton server Docker image builders
 - Add SAMBboxToInstanceMask transform
-- Add ConfigurableValicator
+- Add ConfigurableValidator
 
 Enhancements
 ^^^^^^^^^^^^

diff --git a/notebooks/07_data_explorer.ipynb b/notebooks/07_data_explorer.ipynb
@@ -26,7 +26,7 @@
     "# SPDX-License-Identifier: MIT\n",
     "\n",
     "import datumaro as dm\n",
-    "from datumaro.components.explorer import Explorer\n",
+    "from datumaro.components.algorithms.hash_key_inference.explorer import Explorer\n",
     "from datumaro.components.visualizer import Visualizer"
    ]
   },
@@ -46,7 +46,7 @@
    "metadata": {},
    "source": [
     "### Explore with COCO instance segmentation dataset\n",
-    "To use data exploration, we need to define hash for each dataset. So, set `save_hash` as `True`. The default value is `False`."
+    "To use data exploration, we need to define hash for each dataset. `Explorer` calculates the hash key automatically. If you want to re-use the calculated hash key, please set `save_hashkey_meta` as `True` when exporting the dataset. The default value is `False`."
    ]
   },
   {

diff --git a/rust/Cargo.toml b/rust/Cargo.toml
@@ -11,3 +11,4 @@ pyo3 = "0.19.2"
 serde = { version = "1.0.180", features = ["derive"] }
 serde_json = "1.0.104"
 strum = { version = "0.25", features = ["derive"] }
+tempfile = "3.8.0"