From fa55571de5ed94891beceb6c12e71992fba6ce1a Mon Sep 17 00:00:00 2001 From: Roboto Bot-o Date: Tue, 15 Oct 2024 14:19:45 -0700 Subject: [PATCH] Update SDK to version 0.11.3 --- README.md | 13 +- build-support/lockfiles/python-default.lock | 71 ++++---- examples/BUILD | 7 + examples/fetch-topic-data.ipynb | 190 ++++++++++++++++++++ examples/similarity-search.ipynb | 34 ++-- src/roboto/__init__.py | 2 + src/roboto/domain/actions/trigger_record.py | 3 + src/roboto/domain/datasets/dataset.py | 88 +++++---- src/roboto/domain/events/event.py | 4 +- src/roboto/domain/events/operations.py | 4 +- src/roboto/domain/events/record.py | 2 +- src/roboto/domain/files/file.py | 2 +- src/roboto/domain/files/file_service.py | 8 +- src/roboto/domain/orgs/org.py | 6 +- src/roboto/domain/orgs/org_operations.py | 2 + src/roboto/domain/topics/record.py | 2 +- src/roboto/regionalization.py | 18 ++ src/roboto/version.py | 2 +- 18 files changed, 359 insertions(+), 99 deletions(-) create mode 100644 examples/fetch-topic-data.ipynb create mode 100644 src/roboto/regionalization.py diff --git a/README.md b/README.md index 997b5e9..5a9a534 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,8 @@ It consists of the `roboto` Python module, as well as a `roboto` command line ut If this is your first time using Roboto, we recommend reading the [docs](https://docs.roboto.ai/) and learning the [core concepts](https://docs.roboto.ai/learn/concepts.html). +See below for getting started [examples](#getting-started). + ## Sign up In order to use the Roboto SDK and CLI you'll need to create an account and get an access token. @@ -104,8 +106,8 @@ bag = ds.get_file_by_path("scene57.bag") steering_topic = bag.get_topic("/vehicle_monitor/steering") steering_data = steering_topic.get_data( - start_time=1714513576, - end_time=1714513590, + start_time="1714513576", # "." since epoch + end_time="1714513590", ) ``` @@ -115,8 +117,8 @@ You can also create events: from roboto import Event Event.create( - start_time=1714513580, # seconds since epoch - end_time=1714513590, # seconds since epoch + start_time="1714513580", # "." since epoch + end_time="1714513590", name="Fast Turn", associations = [ steering_topic.to_association() @@ -138,13 +140,14 @@ topics[0].msgpaths[/vehicle_monitor/vehicle_speed.data].max > 20 results = roboto_search.find_files(query) ``` -Coming soon: example notebooks! +See the [notebooks](https://github.com/roboto-ai/roboto-python-sdk/tree/main/examples) directory for complete examples! ## Learn more For more information, check out: * [General Docs](https://docs.roboto.ai/) * [User Guides](https://docs.roboto.ai/user-guides/index.html) +* [Example Notebooks](https://github.com/roboto-ai/roboto-python-sdk/tree/main/examples) * [SDK Reference](https://docs.roboto.ai/reference/python-sdk/roboto/index.html) * [CLI Reference](https://docs.roboto.ai/reference/cli.html) * [About Roboto](https://www.roboto.ai/about) diff --git a/build-support/lockfiles/python-default.lock b/build-support/lockfiles/python-default.lock index 530ea9a..28f2986 100644 --- a/build-support/lockfiles/python-default.lock +++ b/build-support/lockfiles/python-default.lock @@ -137,13 +137,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "c7d2e9d63e31599eeb636c8c5c03a7e108d73b345f064f1c19fdc87b79036a9a", - "url": "https://files.pythonhosted.org/packages/9e/ef/7a4f225581a0d7886ea28359179cb861d7fbcdefad29663fc1167b86f69f/anyio-4.6.0-py3-none-any.whl" + "hash": "6d170c36fba3bdd840c73d3868c1e777e33676a69c3a72cf0a0d5d6d8009b61d", + "url": "https://files.pythonhosted.org/packages/e4/f5/f2b75d2fc6f1a260f340f0e7c6a060f4dd2961cc16884ed851b0d18da06a/anyio-4.6.2.post1-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "137b4559cbb034c477165047febb6ff83f390fc3b20bf181c1fc0a728cb8beeb", - "url": "https://files.pythonhosted.org/packages/78/49/f3f17ec11c4a91fe79275c426658e509b07547f874b14c1a526d86a83fc8/anyio-4.6.0.tar.gz" + "hash": "4c8bc31ccdb51c7f7bd251f51c609e038d63e34219b44aa86e47576389880b4c", + "url": "https://files.pythonhosted.org/packages/9f/09/45b9b7a6d4e45c6bcb5bf61d19e3ab87df68e0601fa8c5293de3542546cc/anyio-4.6.2.post1.tar.gz" } ], "project_name": "anyio", @@ -164,11 +164,12 @@ "sphinx-rtd-theme; extra == \"doc\"", "trio>=0.26.1; extra == \"trio\"", "trustme; extra == \"test\"", + "truststore>=0.9.1; python_version >= \"3.10\" and extra == \"test\"", "typing-extensions>=4.1; python_version < \"3.11\"", "uvloop>=0.21.0b1; (platform_python_implementation == \"CPython\" and platform_system != \"Windows\") and extra == \"test\"" ], "requires_python": ">=3.9", - "version": "4.6.0" + "version": "4.6.2.post1" }, { "artifacts": [ @@ -644,36 +645,36 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "385ca77bf8ea4ab2d97f6e2435bdb29f77d9301e2f7ac796c2f465753c2adf3c", - "url": "https://files.pythonhosted.org/packages/ad/d9/6094eaa18d9c4817df7f413b40eacb43918f46e3432eba45df926c01854d/boto3-1.35.37-py3-none-any.whl" + "hash": "9352f6d61f15c789231a5d608613f03425059072ed862c32e1ed102b17206abf", + "url": "https://files.pythonhosted.org/packages/bd/4b/292d8ab4770d059634fda8141f4898e4d86637bf873ea650d7eb3ce4312b/boto3-1.35.40-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "470d981583885859fed2fd1c185eeb01cc03e60272d499bafe41b12625b158c8", - "url": "https://files.pythonhosted.org/packages/80/8b/31845869fb935b93d1f1a846d2a8e13dc91af4cf03ba701e2068c08b99af/boto3-1.35.37.tar.gz" + "hash": "33c6a7aeab316f7e0b3ad8552afe95a4a10bfd58519d00741c4d4f3047da8382", + "url": "https://files.pythonhosted.org/packages/9a/52/2c5b5f419cd6c7797177bc83fadc4e0d373e71bea565b733f4728b0894ba/boto3-1.35.40.tar.gz" } ], "project_name": "boto3", "requires_dists": [ - "botocore<1.36.0,>=1.35.37", + "botocore<1.36.0,>=1.35.40", "botocore[crt]<2.0a0,>=1.21.0; extra == \"crt\"", "jmespath<2.0.0,>=0.7.1", "s3transfer<0.11.0,>=0.10.0" ], "requires_python": ">=3.8", - "version": "1.35.37" + "version": "1.35.40" }, { "artifacts": [ { "algorithm": "sha256", - "hash": "64f965d4ba7adb8d79ce044c3aef7356e05dd74753cf7e9115b80f477845d920", - "url": "https://files.pythonhosted.org/packages/3a/1c/3057bfd23f56ff11d41db1db8f2c9423381d6e60ae90594921ac2a63be53/botocore-1.35.37-py3-none-any.whl" + "hash": "072cc47f29cb1de4fa77ce6632e4f0480af29b70816973ff415fbaa3f50bd1db", + "url": "https://files.pythonhosted.org/packages/0c/1f/48869905da0090c061b127caf13e722e95d95772d54bba3bc0aedee3ff33/botocore-1.35.40-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "b2b4d29bafd95b698344f2f0577bb67064adbf1735d8a0e3c7473daa59c23ba6", - "url": "https://files.pythonhosted.org/packages/57/a4/bc96ba621c869f723ce4cb4dadb53fdbb821d64ef1146f0749098ef342cf/botocore-1.35.37.tar.gz" + "hash": "547e0a983856c7d7aeaa30fca2a283873c57c07366cd806d2d639856341b3c31", + "url": "https://files.pythonhosted.org/packages/46/96/50aa3079a75dea9fa3e2179121476e4be46f05e9683f8b68db104b8e57cd/botocore-1.35.40.tar.gz" } ], "project_name": "botocore", @@ -685,7 +686,7 @@ "urllib3<1.27,>=1.25.4; python_version < \"3.10\"" ], "requires_python": ">=3.8", - "version": "1.35.37" + "version": "1.35.40" }, { "artifacts": [ @@ -3011,33 +3012,33 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "b499bc07dbdcd3de92b0a8b29fdf592c111276f6a12fe29c30f6c417dd546d12", - "url": "https://files.pythonhosted.org/packages/42/3a/bdf730640ac523229dd6578e8a581795720a9321399de494374afc437ec5/mypy-1.11.2-py3-none-any.whl" + "hash": "fd313226af375d52e1e36c383f39bf3836e1f192801116b31b090dfcd3ec5266", + "url": "https://files.pythonhosted.org/packages/85/fd/2cc64da1ce9fada64b5d023dfbaf763548429145d08c958c78c02876c7f6/mypy-1.12.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "7f9993ad3e0ffdc95c2a14b66dee63729f021968bff8ad911867579c65d13a79", - "url": "https://files.pythonhosted.org/packages/5c/86/5d7cbc4974fd564550b80fbb8103c05501ea11aa7835edf3351d90095896/mypy-1.11.2.tar.gz" + "hash": "060a07b10e999ac9e7fa249ce2bdcfa9183ca2b70756f3bce9df7a92f78a3c0a", + "url": "https://files.pythonhosted.org/packages/53/cb/64043dec34fbcecaced207b077b8e5041e263da43003cc6309c90bc5e26e/mypy-1.12.0-cp310-cp310-musllinux_1_1_x86_64.whl" }, { "algorithm": "sha256", - "hash": "6e658bd2d20565ea86da7d91331b0eed6d2eee22dc031579e6297f3e12c758c8", - "url": "https://files.pythonhosted.org/packages/64/12/aad3af008c92c2d5d0720ea3b6674ba94a98cdb86888d389acdb5f218c30/mypy-1.11.2-cp310-cp310-musllinux_1_1_x86_64.whl" + "hash": "684a9c508a283f324804fea3f0effeb7858eb03f85c4402a967d187f64562469", + "url": "https://files.pythonhosted.org/packages/74/03/5fa6824555460f74873a414c7f42332c219fdfcfbd63b55b2442794b634b/mypy-1.12.0-cp310-cp310-macosx_11_0_arm64.whl" }, { "algorithm": "sha256", - "hash": "d42a6dd818ffce7be66cce644f1dff482f1d97c53ca70908dff0b9ddc120b77a", - "url": "https://files.pythonhosted.org/packages/78/cd/815368cd83c3a31873e5e55b317551500b12f2d1d7549720632f32630333/mypy-1.11.2-cp310-cp310-macosx_10_9_x86_64.whl" + "hash": "6cabe4cda2fa5eca7ac94854c6c37039324baaa428ecbf4de4567279e9810f9e", + "url": "https://files.pythonhosted.org/packages/89/56/20d3136d6904c369422423d267c5ceb312487586cdd81e90bf7e237b67e7/mypy-1.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl" }, { "algorithm": "sha256", - "hash": "41ea707d036a5307ac674ea172875f40c9d55c5394f888b168033177fce47383", - "url": "https://files.pythonhosted.org/packages/dc/08/cdc1fc6d0d5a67d354741344cc4aa7d53f7128902ebcbe699ddd4f15a61c/mypy-1.11.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl" + "hash": "4397081e620dc4dc18e2f124d5e1d2c288194c2c08df6bdb1db31c38cd1fe1ed", + "url": "https://files.pythonhosted.org/packages/93/6d/9751ed6d77b42a5d704224fbadf6f1a18b5ab655c012d17bc8af819a7f06/mypy-1.12.0-cp310-cp310-macosx_10_9_x86_64.whl" }, { "algorithm": "sha256", - "hash": "801780c56d1cdb896eacd5619a83e427ce436d86a3bdf9112527f24a66618fef", - "url": "https://files.pythonhosted.org/packages/f1/27/e18c93a195d2fad75eb96e1f1cbc431842c332e8eba2e2b77eaf7313c6b7/mypy-1.11.2-cp310-cp310-macosx_11_0_arm64.whl" + "hash": "65a22d87e757ccd95cbbf6f7e181e6caa87128255eb2b6be901bb71b26d8a99d", + "url": "https://files.pythonhosted.org/packages/f9/70/196a3339459fe22296ac9a883bbd998fcaf0db3e8d9a54cf4f53b722cad4/mypy-1.12.0.tar.gz" } ], "project_name": "mypy", @@ -3051,7 +3052,7 @@ "typing-extensions>=4.6.0" ], "requires_python": ">=3.8", - "version": "1.11.2" + "version": "1.12.0" }, { "artifacts": [ @@ -4324,13 +4325,13 @@ "artifacts": [ { "algorithm": "sha256", - "hash": "a6a7ee4235a3f944aa1fa2249307708f893fe5717dc603503c6c7969c070fb7c", - "url": "https://files.pythonhosted.org/packages/e5/0c/0e3c05b1c87bb6a1c76d281b0f35e78d2d80ac91b5f8f524cebf77f51049/pyparsing-3.1.4-py3-none-any.whl" + "hash": "93d9577b88da0bbea8cc8334ee8b918ed014968fd2ec383e868fb8afb1ccef84", + "url": "https://files.pythonhosted.org/packages/be/ec/2eb3cd785efd67806c46c13a17339708ddc346cbb684eade7a6e6f79536a/pyparsing-3.2.0-py3-none-any.whl" }, { "algorithm": "sha256", - "hash": "f86ec8d1a83f11977c9a6ea7598e8c27fc5cddfa5b07ea2241edbbde1d7bc032", - "url": "https://files.pythonhosted.org/packages/83/08/13f3bce01b2061f2bbd582c9df82723de943784cf719a35ac886c652043a/pyparsing-3.1.4.tar.gz" + "hash": "cbf74e27246d595d9a74b186b810f6fbb86726dbf3b9532efb343f6d7294fe9c", + "url": "https://files.pythonhosted.org/packages/8c/d5/e5aeee5387091148a19e1145f63606619cb5f20b83fccb63efae6474e7b2/pyparsing-3.2.0.tar.gz" } ], "project_name": "pyparsing", @@ -4338,8 +4339,8 @@ "jinja2; extra == \"diagrams\"", "railroad-diagrams; extra == \"diagrams\"" ], - "requires_python": ">=3.6.8", - "version": "3.1.4" + "requires_python": ">=3.9", + "version": "3.2.0" }, { "artifacts": [ diff --git a/examples/BUILD b/examples/BUILD index cbd62dc..ace2d21 100644 --- a/examples/BUILD +++ b/examples/BUILD @@ -1,5 +1,12 @@ python_sources( name="notebook-utils", + dependencies=[ + # build-support[sdk_examples] extras + "//build-support:3rdparty-dev-tools#ipython", + "//build-support:3rdparty-dev-tools#jupyter", + "//build-support:3rdparty-dev-tools#matplotlib", + "//build-support:3rdparty-dev-tools#pillow", + ], ) resources( diff --git a/examples/fetch-topic-data.ipynb b/examples/fetch-topic-data.ipynb new file mode 100644 index 0000000..c1440f5 --- /dev/null +++ b/examples/fetch-topic-data.ipynb @@ -0,0 +1,190 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f3fc760a-18a3-4428-acf5-8afad7d24294", + "metadata": {}, + "source": [ + "# Roboto SDK - Fetch Topic Data\n", + "\n", + "This notebook shows how to use the Roboto SDK to:\n", + "- List log files and their topics in a dataset\n", + "- Fetch data for a topic and plot some of it\n", + "- Get events created in a dataset\n", + "- Fetch slices of topic data from an event and plot it\n", + "- Render image topic data that corresponds to the same event\n", + "\n", + "You can use the [Python SDK documentation](https://docs.roboto.ai/reference/python-sdk/roboto/domain/index.html) for more details on any of the functions used below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e82da588-9ac5-4d74-9cbb-241e8fd8fcce", + "metadata": {}, + "outputs": [], + "source": [ + "from roboto import Dataset, File, Topic, Event, MessagePath" + ] + }, + { + "cell_type": "markdown", + "id": "ccdbef4f-4908-4b97-afb1-03e2e3718201", + "metadata": {}, + "source": [ + "## List log files and their topics in a dataset\n", + "The dataset used below is public and from a collection of drone racing data. You can see the [dataset](https://app.roboto.ai/datasets/ds_w6lve6tl6f16) and [collection](https://app.roboto.ai/collections/cl_pxlseuhim8ym) on Roboto." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a3641657-8633-4e10-9ff4-8829682bc532", + "metadata": {}, + "outputs": [], + "source": [ + "ds = Dataset.from_id(\"ds_w6lve6tl6f16\")\n", + "files = ds.list_files()\n", + "\n", + "for file in files:\n", + " print(file.relative_path)\n", + " topics = file.get_topics()\n", + " for topic in topics:\n", + " print(topic.name)" + ] + }, + { + "cell_type": "markdown", + "id": "567f0525-45b8-4096-8a31-945a757c52fc", + "metadata": {}, + "source": [ + "## Fetch data for a topic and plot some of it\n", + "In this case, we're getting `/snappy_imu` topic data matching `linear_acceleration` message paths in a log file.\n", + "\n", + "You can also specify `start_time` and `end_time` in `get_data_as_df`. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f0331f9d-6348-4b67-9708-2d40d83b0071", + "metadata": {}, + "outputs": [], + "source": [ + "# Get a specific IMU topic from a log file\n", + "topic = file.get_topic(\"/snappy_imu\")\n", + "\n", + "# Fetch all of the linear_acceleration data in the IMU topic as a dataframe\n", + "df = topic.get_data_as_df([\"linear_acceleration\"])\n", + "\n", + "df.plot()" + ] + }, + { + "cell_type": "markdown", + "id": "8156e78d-4bea-4848-bf1d-852843ec0b13", + "metadata": {}, + "source": [ + "## Get events created in a dataset\n", + "Events might have been created on the dataset itself, or on underlying files, topics and message paths.\n", + "\n", + "In this case, there's just one topic-level event on `/snappy_imu`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52a7b9f1-76c0-4e75-b1f3-71f76796790d", + "metadata": {}, + "outputs": [], + "source": [ + "events = list(Event.get_by_dataset(ds.dataset_id, transitive=True))\n", + "\n", + "# Show details for one event\n", + "event = events[0]\n", + "topic = Topic.from_id(event.topic_ids[0])\n", + "print(f\"{event.event_id} - {event.name} - {topic.name}\")\n", + "print(f\"t0: {event.start_time}\")\n", + "print(f\"tN: {event.end_time}\")" + ] + }, + { + "cell_type": "markdown", + "id": "63108e19-5892-4ae3-9eb9-0019ef4a3503", + "metadata": {}, + "source": [ + "## Fetch slices of topic data from an event and plot it\n", + "Note, this is a small subset of the data plotted previously. Events make it easy to get specific slices of data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8fa22331-0bfd-4c3e-9759-af6d6fbcbecf", + "metadata": {}, + "outputs": [], + "source": [ + "# Fetch the linear_acceleration data associated with the topic event\n", + "df = event.get_data_as_df([\"linear_acceleration\"])\n", + "df.plot()" + ] + }, + { + "cell_type": "markdown", + "id": "f7e8e877-8d2c-4682-aea5-9c970fb37be8", + "metadata": {}, + "source": [ + "## Render image topic data that corresponds to the same event\n", + "The event wasn't specifically created on the image topic, but we can still use the event's start and end time to fetch data from other corresponding topics." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2756abd4-3f32-4876-979c-62afaeb466f4", + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import display, Image\n", + "import io\n", + "import json\n", + "\n", + "# Get camera topic with image data\n", + "camera_topic = file.get_topic(\"/snappy_cam/stereo_l\")\n", + "\n", + "# Fetch image data corresponding to the event start and end time\n", + "camera_data = camera_topic.get_data(\n", + " start_time=event.start_time,\n", + " end_time=event.end_time\n", + ")\n", + "\n", + "# Show the first 5 images\n", + "for i, datum in enumerate(camera_data):\n", + " if i >= 5:\n", + " break\n", + " display(Image(data=datum[\"data\"], width=500))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/similarity-search.ipynb b/examples/similarity-search.ipynb index b3bb287..7effb08 100644 --- a/examples/similarity-search.ipynb +++ b/examples/similarity-search.ipynb @@ -1,5 +1,22 @@ { "cells": [ + { + "cell_type": "markdown", + "id": "ffd60af6-b052-4b97-8bb9-d656857e9db3", + "metadata": {}, + "source": [ + "# Roboto SDK - Similarity Search\n", + "\n", + "This notebook shows how to use the Roboto SDK to:\n", + "- Retrieve topic data corresponding to an event (in this case IMU data in drone racing logs)\n", + "- Find similar signals in other logs that exhibit the same pattern\n", + "- Visualize matching results\n", + "\n", + "You can use the [Python SDK documentation](https://docs.roboto.ai/reference/python-sdk/roboto/domain/index.html) for more details on any of the functions used below.\n", + "\n", + "The data used in this notebook is public and from a collection of drone racing data. You can see the [collection](https://app.roboto.ai/collections/cl_pxlseuhim8ym) on Roboto." + ] + }, { "cell_type": "markdown", "id": "cb4e0010-2301-48ba-acec-921f1a7911ee", @@ -18,12 +35,8 @@ "import roboto\n", "import roboto.query\n", "\n", - "\n", - "roboto_config = roboto.RobotoConfig.from_env(\"prod\")\n", - "roboto_client = roboto.RobotoClient.from_config(roboto_config)\n", "query_client = roboto.query.QueryClient(\n", - " roboto_client=roboto_client,\n", - " owner_org_id=\"og_najtcyyee2qa\" # Drone Racing EU\n", + " owner_org_id=\"og_najtcyyee2qa\" # Drone Racing EU (public data)\n", ")\n", "robotosearch = roboto.RobotoSearch(query_client=query_client)" ] @@ -45,17 +58,14 @@ "source": [ "import roboto.analytics\n", "\n", - "\n", "event = roboto.domain.events.Event.from_id(\n", " \"ev_6funfjngoznn17x3\", \n", - " roboto_client=roboto_client\n", ")\n", "\n", "# This is the topic on which the event was made.\n", "# In this example, it's the \"/snappy_imu\" topic.\n", "source_topic = roboto.Topic.from_id(\n", - " event.topic_ids[0], \n", - " roboto_client=roboto_client\n", + " event.topic_ids[0],\n", ")\n", "topics_to_match_against = robotosearch.find_topics(f\"topic.name = '{source_topic.name}'\")\n", "\n", @@ -88,11 +98,9 @@ "source": [ "from match_visualization_utils import print_match_results\n", "\n", - "\n", "print_match_results(\n", " matches[:5], \n", - " image_topic=\"/snappy_cam/stereo_l\", \n", - " roboto_client=roboto_client\n", + " image_topic=\"/snappy_cam/stereo_l\"\n", ")" ] } @@ -113,7 +121,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.10.12" + "version": "3.10.9" } }, "nbformat": 4, diff --git a/src/roboto/__init__.py b/src/roboto/__init__.py index b964b17..0165bcd 100644 --- a/src/roboto/__init__.py +++ b/src/roboto/__init__.py @@ -131,6 +131,7 @@ ) from .env import RobotoEnv from .http import BatchRequest, RobotoClient +from .regionalization import RobotoRegion from .roboto_search import RobotoSearch from .warnings import ( roboto_default_warning_behavior, @@ -217,6 +218,7 @@ "RobotoClient", "RobotoConfig", "RobotoEnv", + "RobotoRegion", "RobotoSearch", "S3Credentials", "SetActionAccessibilityRequest", diff --git a/src/roboto/domain/actions/trigger_record.py b/src/roboto/domain/actions/trigger_record.py index 2a3ca54..9ec4feb 100644 --- a/src/roboto/domain/actions/trigger_record.py +++ b/src/roboto/domain/actions/trigger_record.py @@ -136,6 +136,9 @@ class TriggerEvaluationRecord(pydantic.BaseModel): evaluation_start: datetime.datetime evaluation_end: typing.Optional[datetime.datetime] = None status: TriggerEvaluationStatus + status_detail: typing.Optional[str] = ( + None # E.g., exception that caused the evaluation to fail + ) outcome: typing.Optional[TriggerEvaluationOutcome] = None outcome_reason: typing.Optional[TriggerEvaluationOutcomeReason] = None cause: typing.Optional[TriggerEvaluationCause] = None diff --git a/src/roboto/domain/datasets/dataset.py b/src/roboto/domain/datasets/dataset.py index 4062715..1ca4dda 100644 --- a/src/roboto/domain/datasets/dataset.py +++ b/src/roboto/domain/datasets/dataset.py @@ -7,8 +7,10 @@ import collections.abc import functools import importlib.metadata +import math import os import pathlib +import threading import typing import pathspec @@ -16,10 +18,7 @@ from ...association import Association from ...auth import Permissions from ...env import RobotoEnv -from ...exceptions import ( - RobotoInternalException, - RobotoNotFoundException, -) +from ...exceptions import RobotoInternalException from ...http import PaginatedList, RobotoClient from ...logging import default_logger from ...paths import excludespec_from_patterns @@ -60,13 +59,19 @@ class Dataset: + UPLOAD_REPORTING_BATCH_COUNT: typing.ClassVar[int] = 10 + """ + Number of batches to break a large upload into for the purpose of reporting progress. + """ + UPLOAD_REPORTING_MIN_BATCH_SIZE: typing.ClassVar[int] = 10 + """ + Minimum number of files that must be uploaded before reporting progress. + """ + __roboto_client: RobotoClient __record: DatasetRecord - __temp_credentials: typing.Optional[DatasetCredentials] = None - __transaction_manifests: dict[str, set[str]] __transaction_completed_unreported_items: dict[str, set[str]] - __manifest_reporting_increments: int = 10 - __manifest_reporting_min_batch_size: int = 10 + __transaction_completed_mutex: threading.Lock @classmethod def create( @@ -144,8 +149,8 @@ def __init__( ) -> None: self.__roboto_client = RobotoClient.defaulted(roboto_client) self.__record = record - self.__transaction_manifests = {} self.__transaction_completed_unreported_items = {} + self.__transaction_completed_mutex = threading.Lock() def __repr__(self) -> str: return self.__record.model_dump_json() @@ -673,38 +678,54 @@ def __list_files_page( def __on_manifest_item_complete( self, transaction_id: str, + total_file_count: int, manifest_item_identifier: str, ) -> None: - if transaction_id not in self.__transaction_manifests: - raise RobotoNotFoundException( - f"Transaction {transaction_id} does not have a manifest" - ) + """ + This method is used as a callback (a "subscriber") for S3 TransferManager, + which is used to upload files to S3. - if transaction_id not in self.__transaction_completed_unreported_items: - self.__transaction_completed_unreported_items[transaction_id] = set() + TransferManager uses thread-based concurrency under the hood, + so any instance state accessed or modified here must be synchronized. + """ + with self.__transaction_completed_mutex: + if transaction_id not in self.__transaction_completed_unreported_items: + self.__transaction_completed_unreported_items[transaction_id] = set() - self.__transaction_completed_unreported_items[transaction_id].add( - manifest_item_identifier - ) + self.__transaction_completed_unreported_items[transaction_id].add( + manifest_item_identifier + ) - if self.__unreported_manifest_items_batch_ready_to_report(transaction_id): - self._flush_manifest_item_completions( - transaction_id=transaction_id, - manifest_items=list( - self.__transaction_completed_unreported_items[transaction_id] - ), + completion_count = len( + self.__transaction_completed_unreported_items[transaction_id] ) - self.__transaction_completed_unreported_items[transaction_id] = set() + if self.__sufficient_uploads_completed_to_report_progress( + completion_count, total_file_count + ): + self._flush_manifest_item_completions( + transaction_id=transaction_id, + manifest_items=list( + self.__transaction_completed_unreported_items[transaction_id] + ), + ) + self.__transaction_completed_unreported_items[transaction_id] = set() - def __unreported_manifest_items_batch_ready_to_report(self, transaction_id): + def __sufficient_uploads_completed_to_report_progress( + self, completion_count: int, total_file_count: int + ): + """ + Determine if there are a sufficient number of files that have already been uploaded + to S3 to report progress to the Roboto Platform. + + If the total count of files to upload is below the reporting threshold, + or if for whatever other reason a batch is not large enough to report progress, + file records will still be finalized as part of the upload finalization routine. + See, e.g., :py:meth:`~roboto.domain.datasets.dataset.Dataset._complete_manifest_transaction` + """ + batch_size = math.ceil(total_file_count / Dataset.UPLOAD_REPORTING_BATCH_COUNT) return ( - len(self.__transaction_completed_unreported_items[transaction_id]) - >= ( - len(self.__transaction_manifests[transaction_id]) - / self.__manifest_reporting_increments - ) - and len(self.__transaction_completed_unreported_items[transaction_id]) - >= self.__manifest_reporting_min_batch_size + completion_count >= batch_size + and completion_count >= Dataset.UPLOAD_REPORTING_MIN_BATCH_SIZE ) def __upload_files_batch( @@ -757,6 +778,7 @@ def __upload_files_batch( on_file_complete=functools.partial( self.__on_manifest_item_complete, transaction_id, + total_file_count, ), progress_monitor=progress_monitor, max_concurrency=8, diff --git a/src/roboto/domain/events/event.py b/src/roboto/domain/events/event.py index eb5e0b1..a1e9429 100644 --- a/src/roboto/domain/events/event.py +++ b/src/roboto/domain/events/event.py @@ -45,8 +45,8 @@ class Event: """ - An event is a "time anchor" which allows you to relate first class Roboto entities (datasets, files, and topics), - as well as a timespan in which they occurred. + An event is a "time anchor" which allows you to relate first class Roboto entities + (datasets, files, topics and message paths), as well as a timespan in which they occurred. """ __roboto_client: RobotoClient diff --git a/src/roboto/domain/events/operations.py b/src/roboto/domain/events/operations.py index 1f2ed19..3d4c741 100644 --- a/src/roboto/domain/events/operations.py +++ b/src/roboto/domain/events/operations.py @@ -20,8 +20,8 @@ class CreateEventRequest(pydantic.BaseModel): associations: list[Association] = pydantic.Field(default_factory=list) """ - Datasets, files, and topics which this event pertains to. At least one must be provided. All referenced - datasets, files, and topics must be owned by the same organization. + Datasets, files, topics and message paths which this event relates to. At least one must be provided. All referenced + datasets, files, topics and message paths must be owned by the same organization. """ description: typing.Optional[str] = None diff --git a/src/roboto/domain/events/record.py b/src/roboto/domain/events/record.py index a876f35..aee6f93 100644 --- a/src/roboto/domain/events/record.py +++ b/src/roboto/domain/events/record.py @@ -19,7 +19,7 @@ class EventRecord(pydantic.BaseModel): associations: list[Association] = pydantic.Field(default_factory=list) """ - Datasets, files, and topics which this event pertains to. + Datasets, files, topics and message paths which this event pertains to. """ created: datetime.datetime diff --git a/src/roboto/domain/files/file.py b/src/roboto/domain/files/file.py index 54d6adf..1cdeb7b 100644 --- a/src/roboto/domain/files/file.py +++ b/src/roboto/domain/files/file.py @@ -97,7 +97,7 @@ def from_path_and_dataset_id( roboto_client: typing.Optional[RobotoClient] = None, ) -> "File": roboto_client = RobotoClient.defaulted(roboto_client) - url_quoted_file_path = urllib.parse.quote_plus(str(file_path)) + url_quoted_file_path = urllib.parse.quote(str(file_path), safe="") record = roboto_client.get( f"v1/files/record/path/{url_quoted_file_path}/association/{dataset_id}", query={"version_id": version_id} if version_id is not None else None, diff --git a/src/roboto/domain/files/file_service.py b/src/roboto/domain/files/file_service.py index 9287557..38aea62 100644 --- a/src/roboto/domain/files/file_service.py +++ b/src/roboto/domain/files/file_service.py @@ -35,13 +35,13 @@ class DynamicCallbackSubscriber(s3_transfer.BaseSubscriber): - __on_done_cb: Optional[Callable[[Any], None]] + __on_done_cb: Optional[Callable[[], None]] __on_progress_cb: Optional[Callable[[Any], None]] __on_queued_cb: Optional[Callable[[Any], None]] def __init__( self, - on_done_cb: Optional[Callable[[Any], None]] = None, + on_done_cb: Optional[Callable[[], None]] = None, on_progress_cb: Optional[Callable[[Any], None]] = None, on_queued_cb: Optional[Callable[[Any], None]] = None, ): @@ -59,7 +59,7 @@ def on_progress(self, future, bytes_transferred, **kwargs): def on_done(self, future, **kwargs): if self.__on_done_cb is not None: - self.__on_done_cb(future) + self.__on_done_cb() class FileService: @@ -244,7 +244,7 @@ def __download_many_files( kwargs={"unit": "file"}, ) - def on_done_cb(future): + def on_done_cb(): progress_monitor.update(1) subscriber = DynamicCallbackSubscriber(on_done_cb=on_done_cb) diff --git a/src/roboto/domain/orgs/org.py b/src/roboto/domain/orgs/org.py index 084d867..f52096a 100644 --- a/src/roboto/domain/orgs/org.py +++ b/src/roboto/domain/orgs/org.py @@ -9,6 +9,7 @@ import urllib.parse from ...http import RobotoClient +from ...regionalization import RobotoRegion from .org_invite import OrgInvite from .org_operations import ( CreateOrgRequest, @@ -33,10 +34,13 @@ def create( cls, name: str, bind_email_domain: bool = False, + data_region: RobotoRegion = RobotoRegion.US_WEST, roboto_client: typing.Optional[RobotoClient] = None, ) -> "Org": roboto_client = RobotoClient.defaulted(roboto_client) - request = CreateOrgRequest(name=name, bind_email_domain=bind_email_domain) + request = CreateOrgRequest( + name=name, bind_email_domain=bind_email_domain, data_region=data_region + ) record = roboto_client.post("v1/orgs", data=request).to_record(OrgRecord) return cls(record=record, roboto_client=roboto_client) diff --git a/src/roboto/domain/orgs/org_operations.py b/src/roboto/domain/orgs/org_operations.py index 4632cde..e7631ce 100644 --- a/src/roboto/domain/orgs/org_operations.py +++ b/src/roboto/domain/orgs/org_operations.py @@ -8,12 +8,14 @@ import pydantic +from ...regionalization import RobotoRegion from .org_records import OrgRoleName, OrgStatus class CreateOrgRequest(pydantic.BaseModel): name: str bind_email_domain: bool = False + data_region: RobotoRegion = RobotoRegion.US_WEST class OrgRecordUpdates(pydantic.BaseModel): diff --git a/src/roboto/domain/topics/record.py b/src/roboto/domain/topics/record.py index 025f435..e872265 100644 --- a/src/roboto/domain/topics/record.py +++ b/src/roboto/domain/topics/record.py @@ -107,7 +107,7 @@ class MessagePathRecord(pydantic.BaseModel): data_type: str """ 'Native'/framework-specific data type of the attribute at this path. - E.g. "float32", "unint8[]", "geometry_msgs/Pose", "string". + E.g. "float32", "uint8[]", "geometry_msgs/Pose", "string". """ message_path: str diff --git a/src/roboto/regionalization.py b/src/roboto/regionalization.py new file mode 100644 index 0000000..9a130a1 --- /dev/null +++ b/src/roboto/regionalization.py @@ -0,0 +1,18 @@ +# Copyright (c) 2024 Roboto Technologies, Inc. +# +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import enum + + +class RobotoRegion(str, enum.Enum): + """ + The geographic region of a Roboto resource. Used when configuring org-level default behavior for data storage, in + order to ensure that data is close to your users. + """ + + US_WEST = "us-west" + US_EAST = "us-east" + EU_CENTRAL = "eu-central" diff --git a/src/roboto/version.py b/src/roboto/version.py index 1a217e3..4a8f140 100644 --- a/src/roboto/version.py +++ b/src/roboto/version.py @@ -1,4 +1,4 @@ -__version__ = "0.11.2" +__version__ = "0.11.3" __all__= ("__version__",) \ No newline at end of file