diff --git a/.gitignore b/.gitignore index 57cb9d418..4f4aad06a 100644 --- a/.gitignore +++ b/.gitignore @@ -50,8 +50,3 @@ coverage.xml # MS stuff *.code-workspace .vscode - -# DSDL generated namespaces -/uavcan/* -/sirius_cyber_corp/* -/test_dsdl_namespace/* diff --git a/.gitmodules b/.gitmodules index a180fa001..1e76eb019 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "public_regulated_data_types_for_testing"] - path = tests/public_regulated_data_types + path = demo/public_regulated_data_types url = https://github.com/UAVCAN/public_regulated_data_types diff --git a/.idea/dictionaries/pavel.xml b/.idea/dictionaries/pavel.xml index 5214a8365..ff61e2b71 100644 --- a/.idea/dictionaries/pavel.xml +++ b/.idea/dictionaries/pavel.xml @@ -255,6 +255,7 @@ toctree todos tradeoff + transcompiling typecheck uart uavcan diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 949a7f56d..18e6f6909 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -16,23 +16,16 @@ When adding new tools and such, please put all their configuration there to keep All shippable entities are located exclusively inside the directory ``pyuavcan/``. The entirety of the directory is shipped. -The directory ``tests/public_regulated_data_types/`` is needed only for testing and documentation building. +The submodule ``demo/public_regulated_data_types/`` is needed only for demo, testing, and documentation building. It should be kept reasonably up-to-date, but remember that it does not affect the final product in any way. We no longer ship DSDL namespaces with code for reasons explained in the user documentation. Please desist from adding any new VCS submodules or subtrees. -The usage demo scripts that are included in the user documentation are located under ``tests/demo/``. -This is probably mildly surprising since one would expect to find docs under ``docs/``, -but it is done this way to facilitate testing and static analysis of the demo scripts. +The demos that are included in the user documentation are located under ``demo/``. Whenever the test suite is run, it tests the demo application as well in order to ensure that it is correct and compatible with the current version of the library -- keeping the docs up-to-date is vitally important. -The CLI tool is auto-tested as well, the tests are located under ``tests/cli/``. -It's somewhat trickier than with the rest of the code because it requires us to -launch new processes and keep track of their code coverage metrics; -the details are explained in a dedicated section. - There are major automation scripts located in the source root directory. You will need them if you are developing the library; please open them and read the comments inside to understand how they work and how to use them. @@ -45,7 +38,7 @@ Third-party dependencies The general rule is that external dependencies are to be avoided unless doing so would increase the complexity of the codebase considerably. -There are three kinds of 3rd-party dependencies used by this library: +There are two kinds of 3rd-party dependencies used by this library: - **Core dependencies.** Those are absolutely required to use the library. The list of core deps contains two libraries: Nunavut and NumPy, and it is probably not going to be extended ever @@ -59,9 +52,6 @@ There are three kinds of 3rd-party dependencies used by this library: conventions in the user documentation and in ``setup.cfg``. When developing new transports or media sub-layers, try to avoid adding new dependencies. -- **Other dependencies.** Those are needed for some minor optional components and features of the library, - such as the CLI tool. - Coding conventions ------------------ @@ -187,7 +177,7 @@ The scanner should not be run before the full general test suite since it relies When writing tests, aim to cover at least 90% of branches, excepting the DSDL generated packages (at least for now) (the DSDL test data is synthesized at run time). -Ensure that your tests do not emit any errors or warnings into the CLI output upon successful execution, +Ensure that your tests do not emit any errors or warnings into stderr output upon successful execution, because that may distract the developer from noticing true abnormalities (you may use ``caplog.at_level('CRITICAL')`` to suppress undesirable output). @@ -214,22 +204,6 @@ Certain tests require real-time execution. If they appear to be failing with timeout errors and such, consider re-running them on a faster system. It is recommended to run the test suite with at least 2 GB of free RAM and an SSD. -Auto-tests may spawn new processes, e.g., to test the CLI tool. In order to keep their code coverage measured, -we have put the coverage setup code into a special module ``sitecustomize.py``, which is auto-imported -every time a new Python interpreter is started (as long as the module's path is in ``PYTHONPATH``, of course). -Hence, every invocation of Python made during testing is coverage-tracked, which is great. -This is why we don't invoke ``coverage`` manually when running tests. -After the tests are executed, we end up with some dozen or more of ``.coverage*`` files scattered across the -source directories. -The scattered coverage files are then located automatically and combined into one file, -which is then analyzed by report generators and other tools like SonarQube. - -When tests that spawn new processes fail, they may leave their children running in the background, -which may adversely influence other tests that are executed later, -so an error in one test may crash a dozen of unrelated ones invoked afterwards. -You need to be prepared for that and always start analyzing the test report starting with the first failure. -Ideally, though, this should be fixed by adding robust cleanup logic for each test. - Some of the components of the library and of the test suite require DSDL packages to be generated. Those must be dealt with carefully as it needs to be ensured that the code that requires generated packages to be available is not executed until they are generated. @@ -239,7 +213,7 @@ and other higher-level components are tested against them. At least the following locations should be checked first: - ``tests/presentation`` -- generic presentation layer test cases. -- ``tests/cli`` -- CLI and demo test cases. +- ``tests/demo`` -- demo test cases. - The list may not be exhaustive, please grep the sources to locate all relevant modules. Many tests rely on the DSDL-generated packages being available for importing. @@ -265,14 +239,6 @@ Normally, this should be done a few months after a new version of CPython is rel If the CI/CD pipelines pass, you are all set. -Debugging ---------- - -When debugging argument parsing issues in the CLI, -you won't see any stacktrace unless verbose logging is enabled before the argument parser is constructed. -To work around that, use the environment variable `PYUAVCAN_LOGLEVEL` (see the user docs for details). - - Releasing --------- @@ -300,3 +266,5 @@ Make sure to mark it as a source directory to enable code completion and type an (for PyCharm: right click -> Mark Directory As -> Sources Root). Configure the IDE to remove trailing whitespace on save in the entire file. +Or, even better, configure a File Watcher to run Black on save +(make sure to disable running it on external file changes though). diff --git a/README.md b/README.md index ce57ebf93..cadec6594 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,6 @@ Full-featured UAVCAN stack in Python PyUAVCAN is a full-featured implementation of the UAVCAN protocol stack intended for non-embedded, user-facing applications such as GUI software, diagnostic tools, automation scripts, prototypes, and various R&D cases. -PyUAVCAN consists of a Python library (package) and a simple CLI tool for basic diagnostics and shell script automation. PyUAVCAN aims to support all features and transport layers of UAVCAN, be portable across all major platforms supporting Python, @@ -31,3 +30,6 @@ The acronym *UAVCAN* stands for ***Uncomplicated Application-level Vehicular Com **READ THE DOCS: [pyuavcan.readthedocs.io](https://pyuavcan.readthedocs.io/)** **Ask questions: [forum.uavcan.org](https://forum.uavcan.org/)** + +*See also: [**U-tool**](https://github.com/UAVCAN/u) -- a CLI tool for diagnostics and management of +UAVCAN networks built on top of PyUAVCAN.* diff --git a/clean.sh b/clean.sh index e7bb557b8..008aa3c01 100755 --- a/clean.sh +++ b/clean.sh @@ -2,9 +2,6 @@ rm -rf dist build ./*.egg-info .coverage* htmlcov .*cache .*_generated *.log *.tmp &> /dev/null -# DSDL-generated packages -rm -rf uavcan sirius_cyber_corp test_dsdl_namespace &> /dev/null - pushd docs || exit 1 rm -rf _build .coverage* .*_generated popd || exit 1 diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 000000000..114f05723 --- /dev/null +++ b/demo/README.md @@ -0,0 +1,7 @@ +PyUAVCAN demo application +========================= + +This directory contains the demo application. +It is invoked and verified by the integration test suite along with the main library codebase. + +Please refer to the official library documentation for details about this demo. diff --git a/tests/dsdl/namespaces/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan b/demo/custom_data_types/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan rename to demo/custom_data_types/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan diff --git a/tests/dsdl/namespaces/sirius_cyber_corp/PointXY.1.0.uavcan b/demo/custom_data_types/sirius_cyber_corp/PointXY.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/sirius_cyber_corp/PointXY.1.0.uavcan rename to demo/custom_data_types/sirius_cyber_corp/PointXY.1.0.uavcan diff --git a/tests/demo/demo_app.py b/demo/demo_app.py similarity index 91% rename from tests/demo/demo_app.py rename to demo/demo_app.py index 432b70139..b976dde7b 100755 --- a/tests/demo/demo_app.py +++ b/demo/demo_app.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -# PyUAVCAN demo application. This file is included in the user documentation, please keep it tidy. +# PyUAVCAN demo application. # # Distributed under CC0 1.0 Universal (CC0 1.0) Public Domain Dedication. To the extent possible under law, the # UAVCAN Consortium has waived all copyright and related or neighboring rights to this work. @@ -45,18 +45,18 @@ import sirius_cyber_corp # This is our vendor-specific root namespace. Custom data types. import pyuavcan.application # The application module requires the standard types from the root namespace "uavcan". except (ImportError, AttributeError): - src_dir = os.path.abspath(os.path.dirname(__file__)) - # Generate our vendor-specific namespace. It may make use of the standard data types (most namespaces do, + src_dir = pathlib.Path(__file__).resolve().parent + # Generate our application-specific namespace. It may make use of the standard data types (most namespaces do, # because the standard root namespace contains important basic types), so we include it in the lookup path set. # The paths are hard-coded here for the sake of conciseness. pyuavcan.dsdl.generate_package( - root_namespace_directory=os.path.join(src_dir, "../dsdl/namespaces/sirius_cyber_corp/"), - lookup_directories=[os.path.join(src_dir, "../public_regulated_data_types/uavcan")], + root_namespace_directory=src_dir / "custom_data_types/sirius_cyber_corp", + lookup_directories=[src_dir / "public_regulated_data_types/uavcan/"], output_directory=dsdl_generated_dir, ) # Generate the standard namespace. The order actually doesn't matter. pyuavcan.dsdl.generate_package( - root_namespace_directory=os.path.join(src_dir, "../public_regulated_data_types/uavcan"), + root_namespace_directory=src_dir / "public_regulated_data_types/uavcan/", output_directory=dsdl_generated_dir, ) # Okay, we can try importing again. We need to clear the import cache first because Python's import machinery @@ -73,7 +73,7 @@ class DemoApplication: - def __init__(self): + def __init__(self) -> None: # The interface to run the demo against is selected via the environment variable with a default option provided. # Virtual CAN bus is supported only on GNU/Linux, but other interfaces used here should be compatible # with at least Windows as well. @@ -88,6 +88,7 @@ def __init__(self): # For example, anonymous node cannot be a server, since without an ID it cannot be addressed. # Here, we assign a node-ID statically, because this is a simplified demo. # Most applications would need this to be configurable, some may support the PnP node-ID allocation protocol. + transport: pyuavcan.transport.Transport if interface_kind == "udp" or not interface_kind: # This is the default. # The UDP/IP transport in this example runs on the local loopback interface, so no setup is needed. # The UDP transport requires us to specify the IP address; the node-ID equals the value of several least @@ -122,6 +123,7 @@ def __init__(self): media_1 = pyuavcan.transport.can.media.socketcan.SocketCANMedia(f"vcan1", mtu=32) media_2 = pyuavcan.transport.can.media.socketcan.SocketCANMedia(f"vcan2", mtu=64) # All transports in a redundant group MUST share the same node-ID. + assert isinstance(transport, pyuavcan.transport.redundant.RedundantTransport) transport.attach_inferior(pyuavcan.transport.can.CANTransport(media_0, local_node_id=42)) transport.attach_inferior(pyuavcan.transport.can.CANTransport(media_1, local_node_id=42)) transport.attach_inferior(pyuavcan.transport.can.CANTransport(media_2, local_node_id=42)) @@ -134,6 +136,7 @@ def __init__(self): # may be observed with wired+wireless links used concurrently; see https://forum.uavcan.org/t/557. # All transports in a redundant group MUST share the same node-ID. transport = pyuavcan.transport.redundant.RedundantTransport() + assert isinstance(transport, pyuavcan.transport.redundant.RedundantTransport) transport.attach_inferior(pyuavcan.transport.udp.UDPTransport("127.0.0.42")) transport.attach_inferior( pyuavcan.transport.serial.SerialTransport("socket://localhost:50905", local_node_id=42) @@ -163,7 +166,7 @@ def __init__(self): self._node = pyuavcan.application.Node(presentation, node_info) # Published heartbeat fields can be configured trivially by assigning them on the heartbeat publisher instance. - self._node.heartbeat_publisher.mode = uavcan.node.Mode_1_0.OPERATIONAL + self._node.heartbeat_publisher.mode = uavcan.node.Mode_1_0.OPERATIONAL # type: ignore # The vendor-specific status code is the two least significant decimal digits of the local process' PID. self._node.heartbeat_publisher.vendor_specific_status_code = os.getpid() % 100 @@ -219,10 +222,10 @@ async def _serve_linear_least_squares_request( self._pub_diagnostic_record.publish_soon(diagnostic_msg) # This is just the business logic. - sum_x = sum(map(lambda p: p.x, request.points)) - sum_y = sum(map(lambda p: p.y, request.points)) - a = sum_x * sum_y - len(request.points) * sum(map(lambda p: p.x * p.y, request.points)) - b = sum_x * sum_x - len(request.points) * sum(map(lambda p: p.x ** 2, request.points)) + sum_x = sum(map(lambda p: p.x, request.points)) # type: ignore + sum_y = sum(map(lambda p: p.y, request.points)) # type: ignore + a = sum_x * sum_y - len(request.points) * sum(map(lambda p: p.x * p.y, request.points)) # type: ignore + b = sum_x * sum_x - len(request.points) * sum(map(lambda p: p.x ** 2, request.points)) # type: ignore try: slope = a / b y_intercept = (sum_y - slope * sum_x) / len(request.points) @@ -241,7 +244,7 @@ async def _serve_linear_least_squares_request( self._pub_diagnostic_record.publish_soon( uavcan.diagnostic.Record_1_1( severity=uavcan.diagnostic.Severity_1_0(uavcan.diagnostic.Severity_1_0.INFO), - text=f'Solution for {",".join(f"({p.x},{p.y})" for p in request.points)}: {slope}, {y_intercept}', + text=f"Solution for {','.join(f'({p.x},{p.y})' for p in request.points)}: {slope}, {y_intercept}", ) ) return sirius_cyber_corp.PerformLinearLeastSquaresFit_1_0.Response(slope=slope, y_intercept=y_intercept) @@ -295,9 +298,7 @@ async def _handle_temperature( ) ): print( - "Diagnostic message could not be sent in", - self._pub_diagnostic_record.send_timeout, - "seconds", + f"Diagnostic publication timed out in {self._pub_diagnostic_record.send_timeout} seconds", file=sys.stderr, ) @@ -309,14 +310,17 @@ async def _handle_temperature( async def list_tasks_periodically() -> None: """Print active tasks periodically for demo purposes.""" while True: - print( - "\nRunning tasks:\n" + "\n".join(f"{i:4}: {t.get_coro()}" for i, t in enumerate(asyncio.all_tasks())), - file=sys.stderr, - ) + if sys.version_info >= (3, 8): # The task introspection API we use is not available before Python 3.8 + print( + "\nRunning tasks:\n" + + "\n".join(f"{i:4}: {t.get_coro()}" for i, t in enumerate(asyncio.all_tasks())), + file=sys.stderr, + ) + else: + print(f"\nRunning {len(asyncio.all_tasks())} tasks") await asyncio.sleep(10) - if sys.version_info >= (3, 8): # The task introspection API we use is not available before Python 3.8 - asyncio.get_event_loop().create_task(list_tasks_periodically()) + asyncio.get_event_loop().create_task(list_tasks_periodically()) # The node and PyUAVCAN objects have created internal tasks, which we need to run now. # In this case we want to automatically stop and exit when no tasks are left to run. diff --git a/tests/public_regulated_data_types b/demo/public_regulated_data_types similarity index 100% rename from tests/public_regulated_data_types rename to demo/public_regulated_data_types diff --git a/docs/conf.py b/docs/conf.py index 67e541d99..46d0c7783 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,7 +30,7 @@ # The generated files are not documented, but they must be importable to import the target package. DSDL_GENERATED_ROOT = REPOSITORY_ROOT / ".test_dsdl_generated" -PUBLIC_REGULATED_DATA_TYPES_ROOT = REPOSITORY_ROOT / "tests" / "public_regulated_data_types" +PUBLIC_REGULATED_DATA_TYPES_ROOT = REPOSITORY_ROOT / "demo" / "public_regulated_data_types" sys.path.insert(0, str(REPOSITORY_ROOT)) sys.path.insert(0, str(DSDL_GENERATED_ROOT)) diff --git a/docs/index.rst b/docs/index.rst index 3bc6a2f98..c8fbb871d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -20,7 +20,6 @@ Contents pages/architecture pages/installation - pages/cli pages/demo pages/faq pages/api diff --git a/docs/pages/architecture.rst b/docs/pages/architecture.rst index 8a42eddc7..8e8f42b2e 100644 --- a/docs/pages/architecture.rst +++ b/docs/pages/architecture.rst @@ -9,8 +9,6 @@ Overview PyUAVCAN is a full-featured implementation of the `UAVCAN protocol stack `_ intended for non-embedded, user-facing applications such as GUI software, diagnostic tools, automation scripts, prototypes, and various R&D cases. -PyUAVCAN consists of a Python library (package) and a simple :abbr:`CLI (command line interface)` -tool for basic diagnostics and shell script automation. It is designed to support **GNU/Linux**, **MS Windows**, and **macOS** as first-class target platforms. The reader should understand the basics of `UAVCAN `_ and be familiar with @@ -329,9 +327,7 @@ For more information, read the API docs for :mod:`pyuavcan.util`. Command-line tool ----------------- -The command-line tool named ``pyuavcan`` (like the library) -can be installed as described in the :ref:`installation` chapter. -Run ``pyuavcan --help`` (or ``python -m pyuavcan --help``) -to see the usage documentation, or read the :ref:`cli` chapter. - -The tool also serves as a (somewhat convoluted) library usage demo. +There is an independent but related project that is built on top of PyUAVCAN: +`Yakut `_. +It is a command-line interface utility for diagnostics and management of UAVCAN networks. +Consider it as an extensive collection of practical usage examples for PyUAVCAN. diff --git a/docs/pages/cli.rst b/docs/pages/cli.rst deleted file mode 100644 index db3bcda32..000000000 --- a/docs/pages/cli.rst +++ /dev/null @@ -1,16 +0,0 @@ -.. _cli: - -Command-line tool -================= - -The command-line tool can be invoked using its full name ``pyuavcan``, using the alias ``uvc``, -or by explicitly invoking the Python package as ``python -m pyuavcan``. - -There is an unlisted optional dependency ``coloredlogs``. -As the name suggests, if this library is installed, the log messages emitted into stderr by the CLI tool -will be nicely colored. - -The information contained below can also be accessed via ``--help``. - -.. computron-injection:: - :filename: synth/cli_help.py diff --git a/docs/pages/demo.rst b/docs/pages/demo.rst index 2bc2e244f..4f5c7601b 100644 --- a/docs/pages/demo.rst +++ b/docs/pages/demo.rst @@ -23,13 +23,13 @@ The referenced DSDL definitions are provided below. ``sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0``: -.. literalinclude:: /../tests/dsdl/namespaces/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan +.. literalinclude:: /../demo/custom_data_types/sirius_cyber_corp/PerformLinearLeastSquaresFit.1.0.uavcan :linenos: ``sirius_cyber_corp.PointXY.1.0``: -.. literalinclude:: /../tests/dsdl/namespaces/sirius_cyber_corp/PointXY.1.0.uavcan +.. literalinclude:: /../demo/custom_data_types/sirius_cyber_corp/PointXY.1.0.uavcan :linenos: @@ -40,31 +40,35 @@ The demo relies on the custom data types presented above. In order to run the demo, please copy-paste its source code into a file on your computer and update the DSDL paths to match your environment. -.. literalinclude:: /../tests/demo/demo_app.py +.. literalinclude:: /../demo/demo_app.py :linenos: -Evaluating the demo using the command-line tool ------------------------------------------------ +Evaluating the demo using Yakut command-line tool +------------------------------------------------- -Generating data type packages from DSDL -+++++++++++++++++++++++++++++++++++++++ +`Yakut `_ is a simple CLI tool for diagnostics and management of UAVCAN networks +built on PyUAVCAN. +Please refer to Yakut docs to see how to get it running on your system. -First, we need to make sure that the required DSDL-generated packages are available for the command-line tool. -Suppose that the application-specific data types listed above are located at ``../dsdl/namespaces/``, -and that instead of using a local copy of the public regulated data types we prefer to download them from the -repository. This is the command: + +Compiling DSDL +++++++++++++++ + +We need to compile DSDL namespaces before using them with Yakut. +Suppose that the application-specific data types listed above are located under ``custom_data_types``, +and the public regulated data types are under ``public_regulated_data_types``. +This is the command: .. code-block:: sh - uvc dsdl-gen-pkg ../dsdl/namespaces/sirius_cyber_corp/ https://github.com/UAVCAN/public_regulated_data_types/archive/master.zip + yakut compile custom_data_types/sirius_cyber_corp public_regulated_data_types/uavcan + +Outputs are stored in the current working directory, so now we can use them. +If you decided to change the working directory or move the compilation outputs, +make sure to update the ``YAKUT_PATH`` environment variable. -That's it. -The DSDL-generated packages have been stored in the current working directory, so now we can use them. -If you decided to change the working directory, please make sure to update the ``PYTHONPATH`` environment -variable to include the path where the generated packages are stored, otherwise you won't be able to import them. -Alternatively, you can just move the generated packages to a new location (they are location-invariant) -or just generate them anew where needed. +This command is actually a thin wrapper over the `Nunavut DSDL transpiler `_. If you want to know what exactly has been done, rerun the command with ``-v`` (V for Verbose). As always, use ``--help`` to get the full usage information. @@ -74,26 +78,22 @@ Configuring the transport +++++++++++++++++++++++++ The commands shown later have to be instructed to use the same transport interface as the demo. -In this example we configure the transport using the environment variable ``PYUAVCAN_CLI_TRANSPORT``, -but it is also possible to use the ``--tr`` command line argument if found more convenient +In this example we configure the transport using the environment variable ``YAKUT_TRANSPORT``, +but it is also possible to use the ``--transport`` command line argument if found more convenient (the syntax is identical). -Please use one of the following transport configuration expressions depending on your demo configuration: +Use one of the following initialization expressions depending on your demo configuration: -- ``"UDP('127.0.0.111')"`` -- - UDP/IP transport on localhost. Local node-ID 111. +- ``"UDP('127.0.0.111')"`` -- UDP/IP on loopback. Local node-ID 111. - ``"Serial('socket://loopback:50905',111)"`` -- - serial transport emulated over a TCP/IP tunnel instead of a real serial port (use Ncat for TCP connection brokering). + UAVCAN/serial emulated over a TCP/IP tunnel instead of a real serial port (use Ncat for TCP connection brokering). Local node-ID 111. - ``"CAN(can.media.socketcan.SocketCANMedia('vcan0',8),111)"`` -- - virtual CAN bus via SocketCAN (GNU/Linux systems only). - Local node-ID 111. + virtual CAN bus via SocketCAN (GNU/Linux systems only). Local node-ID 111. -Redundant transports can be configured by specifying multiple comma-separated expressions (bracketed list is also ok) -(or by specifying the ``--tr`` option more than once if the command line arguments are used instead -of the environment variable): +Redundant transports can be configured by specifying multiple comma-separated expressions: - ``"UDP('127.0.0.111'), Serial('socket://loopback:50905',111)"`` -- dissimilar double redundancy, UDP plus serial. @@ -101,21 +101,17 @@ of the environment variable): - ``"CAN(can.media.socketcan.SocketCANMedia('vcan0',8),111), CAN(can.media.socketcan.SocketCANMedia('vcan1',32),111), CAN(can.media.socketcan.SocketCANMedia('vcan2',64),111)"`` -- triple redundant CAN bus, classic CAN with CAN FD. -Specifying a single transport using the list notation is also acceptable -- -this case is handled as if there was no list notation used: ``[a] == a``. -For more info on command line arguments, see chapter :ref:`cli`. - -If you are using bash/sh/zsh or similar, the syntax to set the variable is: +Complete example if you are using bash/sh/zsh or similar: .. code-block:: sh - export PYUAVCAN_CLI_TRANSPORT="Loopback(None)" # Using LoopbackTransport as an example + export YAKUT_TRANSPORT="UDP('127.0.0.111')" If you are using PowerShell: .. code-block:: ps1 - $env:PYUAVCAN_CLI_TRANSPORT="Loopback(None), Loopback(None)" + $env:YAKUT_TRANSPORT="UDP('127.0.0.111')" Running the application @@ -126,8 +122,8 @@ To listen to the demo's heartbeat or its diagnostics, run the following commands .. code-block:: sh - uvc sub uavcan.node.Heartbeat.1.0 --with-metadata --count=3 - uvc sub uavcan.diagnostic.Record.1.1 --with-metadata + yakut sub uavcan.node.Heartbeat.1.0 --count=3 + yakut sub uavcan.diagnostic.Record.1.1 The latter may not output anything because the demo application is not doing anything interesting, so it has nothing to report. @@ -135,17 +131,17 @@ Keep the command running, and open a yet another terminal, whereat run this: .. code-block:: sh - uvc call 42 123.sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0 '{points: [{x: 10, y: 1}, {x: 20, y: 2}]}' + yakut call 42 123.sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0 'points: [{x: 10, y: 1}, {x: 20, y: 2}]' Once you've executed the last command, you should see a diagnostic message being emitted in the other terminal. Now let's publish temperature: .. code-block:: sh - uvc pub 12345.uavcan.si.sample.temperature.Scalar.1.0 '{kelvin: 123.456}' --count=2 + yakut pub 12345.uavcan.si.sample.temperature.Scalar.1.0 '{kelvin: 123.456}' --count=2 You will see the demo application emit two more diagnostic messages. If you want to see what exactly is happening under the hood, -set the environment variable ``PYUAVCAN_LOGLEVEL=DEBUG`` before starting the process. +export the environment variable ``PYUAVCAN_LOGLEVEL=DEBUG`` before starting the process. This will slow down the library significantly. diff --git a/docs/pages/synth/cli_help.py b/docs/pages/synth/cli_help.py deleted file mode 100755 index 8f6b1f45b..000000000 --- a/docs/pages/synth/cli_help.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -""" -I've drafted up this custom hack instead of using sphinx-argparse because it's broken (generated ReST is -not syntactically correct) and does not support parallel build. -""" - -import textwrap -import subprocess - -# noinspection PyProtectedMember -import pyuavcan._cli as cli - - -HEADER_SUFFIX = "\n" + "." * 80 + "\n\n" - - -def indent(text: str) -> str: - return textwrap.indent(text, " " * 3) - - -def print_output(command_arguments: str) -> None: - print("::", end="\n\n") - print(indent(f"$ pyuavcan {command_arguments}")) - print( - indent(subprocess.check_output(f"python -m pyuavcan {command_arguments}", encoding="utf8", shell=True)).replace( - "__main__.py ", "pyuavcan ", 1 - ), - end="\n\n", - ) - - -print("General help" + HEADER_SUFFIX) -print_output("--version") -print_output("--help") - - -for cls in cli.commands.get_available_command_classes(): - cmd = cls() - print(f"Subcommand ``{cmd.names[0]}``" + HEADER_SUFFIX) - print_output(f"{cmd.names[0]} --help") diff --git a/pyproject.toml b/pyproject.toml index 2c22b6ef4..7acac5bf6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,8 +2,7 @@ line-length = 120 target-version = ['py37'] include = ''' -(pyuavcan|tests)/.*\.pyi?$ -''' -exclude = ''' -tests/public_regulated_data_types/.* +((pyuavcan|tests)/.*\.pyi?$) +| +(demo/[a-z0-9_]+\.py$) ''' diff --git a/pyuavcan/VERSION b/pyuavcan/VERSION index 92e1dda44..c09a4f3ac 100644 --- a/pyuavcan/VERSION +++ b/pyuavcan/VERSION @@ -1 +1 @@ -1.1.0.dev6 +1.1.0.dev7 diff --git a/pyuavcan/__init__.py b/pyuavcan/__init__.py index 07fbf4154..16a0da032 100644 --- a/pyuavcan/__init__.py +++ b/pyuavcan/__init__.py @@ -23,12 +23,6 @@ There are no internal (hidden) API between the submodules; they rely only on each other's public API. -CLI tool invocation -+++++++++++++++++++ - -The module can be run as ``python -m pyuavcan`` to invoke the CLI tool. - - Log level override ++++++++++++++++++ diff --git a/pyuavcan/__main__.py b/pyuavcan/__main__.py deleted file mode 100644 index 3da89123e..000000000 --- a/pyuavcan/__main__.py +++ /dev/null @@ -1,8 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -if __name__ == "__main__": - from pyuavcan import _cli - - _cli.main() diff --git a/pyuavcan/_cli/__init__.py b/pyuavcan/_cli/__init__.py deleted file mode 100644 index 6cc4b1bbd..000000000 --- a/pyuavcan/_cli/__init__.py +++ /dev/null @@ -1,10 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -from ._main import main as main - -# noinspection PyCompatibility -from . import commands as commands - -from .commands import DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL as DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL diff --git a/pyuavcan/_cli/_main.py b/pyuavcan/_cli/_main.py deleted file mode 100644 index 253f5fcd9..000000000 --- a/pyuavcan/_cli/_main.py +++ /dev/null @@ -1,194 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import os -import sys -import time -import typing -import logging -import argparse - -# noinspection PyCompatibility -from . import commands - -_logger = logging.getLogger(__name__) - -_LOG_FORMAT = "%(asctime)s %(process)5d %(levelname)-8s %(name)s: %(message)s" - - -def main() -> None: - logging.basicConfig(format=_LOG_FORMAT) # Using the default log level; it will be overridden later. - - try: - exit(_main_impl()) - except KeyboardInterrupt: - _logger.info("Interrupted") - _logger.debug("Stack trace where the program has been interrupted", exc_info=True) - exit(1) - except AssertionError: - raise # Re-raise directly in order to have the stack trace printed. The user is not expected to see this. - except Exception as ex: - print("Error: %s:" % type(ex).__name__, ex, file=sys.stderr) - _logger.info("Unhandled exception: %s", ex, exc_info=True) - exit(1) - - -def _main_impl() -> int: - command_instances: typing.Sequence[commands.Command] = [cls() for cls in commands.get_available_command_classes()] - - args = _construct_argument_parser(command_instances).parse_args() - - _configure_logging(args.verbose) - - _logger.debug("Available commands: %s", command_instances) - _logger.debug("Parsed args: %s", args) - - # It is a common use case when the user generates DSDL packages in the current directory and then runs the CLI - # tool in it. Do not require the user to manually export PYTHONPATH=. by extending it with the CWD automatically. - sys.path.append(os.getcwd()) - _logger.debug("sys.path: %r", sys.path) - - if hasattr(args, "func"): - started_at = time.monotonic() - try: - result = args.func(args) - except ImportError as ex: - # If the application submodule fails to import with an import error, a DSDL data type package - # probably needs to be generated first, which we suggest the user to do. - from .commands.dsdl_generate_packages import DSDLGeneratePackagesCommand - - raise ImportError(DSDLGeneratePackagesCommand.make_usage_suggestion_text(ex.name or "")) - - _logger.debug("Command executed in %.1f seconds", time.monotonic() - started_at) - assert isinstance(result, int) - return result - else: - print( - "No command specified, nothing to do. Run with --help for usage help. " - "Online support: https://forum.uavcan.org.", - file=sys.stderr, - ) - print("Available commands:", file=sys.stderr) - for cmd in command_instances: - text = f"\t{cmd.names[0]}" - if len(cmd.names) > 1: - text += f' (aliases: {", ".join(cmd.names[1:])})' - print(text, file=sys.stderr) - return 1 - - -def _construct_argument_parser(command_instances: typing.Sequence[commands.Command]) -> argparse.ArgumentParser: - from pyuavcan import __version__ - - # noinspection PyTypeChecker - root_parser = argparse.ArgumentParser( - formatter_class=argparse.RawTextHelpFormatter, - description=r""" - __ __ _______ __ __ _______ _______ __ __ - | | | | / _ \ | | | | / ____| / _ \ | \ | | - | | | | | |_| | | | | | | | | |_| | | \| | - | |_| | | _ | \ \_/ / | |____ | _ | | |\ | - \_______/ |__| |__| \_____/ \_______| |__| |__| |__| \__| - | | | | | | - ----o------o------------o---------o------o---------o------- - -PyUAVCAN CLI -- a command line tool for diagnostics and management of UAVCAN networks. -PyUAVCAN is a Python library implementing the UAVCAN stack for high-level operating systems (GNU/Linux, Windows, macOS) -supporting different transport protocols (UAVCAN/CAN, UAVCAN/UDP/IP, UAVCAN/serial, etc). - -This tool is designed for use either directly by humans or from automation scripts. - -Read the docs: https://pyuavcan.readthedocs.io -Ask questions: https://forum.uavcan.org -""".strip( - "\r\n" - ), - ) - - # Register common arguments - root_parser.add_argument( - "--version", - "-V", - action="version", - version=f"%(prog)s {__version__}", - help=""" -Print the PyUAVCAN version string and exit. The tool is versioned synchronously with the PyUAVCAN library. -""".strip(), - ) - root_parser.add_argument( - "--verbose", - "-v", - action="count", - help="Increase the verbosity of the output. Twice for extra verbosity.", - ) - - # Register commands - subparsers = root_parser.add_subparsers() - for cmd in command_instances: - if cmd.examples: - epilog = "Examples:\n" + cmd.examples - else: - epilog = "" - - parser = subparsers.add_parser( - cmd.names[0], - help=cmd.help, - epilog=epilog, - aliases=cmd.names[1:], - formatter_class=argparse.RawTextHelpFormatter, - ) - cmd.register_arguments(parser) - for sf in cmd.subsystem_factories: - sf.register_arguments(parser) - - parser.set_defaults(func=_make_executor(cmd)) - - return root_parser - - -def _make_executor(cmd: commands.Command) -> typing.Callable[[argparse.Namespace], int]: - def execute(args: argparse.Namespace) -> int: - subsystems: typing.List[object] = [] - for sf in cmd.subsystem_factories: - try: - ss = sf.construct_subsystem(args) - except Exception as ex: - raise RuntimeError( - f"Subsystem factory {type(sf).__name__!r} for command {cmd.names[0]!r} has failed: {ex}" - ) - else: - subsystems.append(ss) - _logger.debug("Invoking %r with subsystems %r and arguments %r", cmd, subsystems, args) - return cmd.execute(args, subsystems) - - return execute - - -def _configure_logging(verbosity_level: int) -> None: - """ - Until this function is invoked we're running the bootstrap default configuration. - This function changes the configuration to use the correct production settings as specified. - """ - log_level = { - 0: logging.WARNING, - 1: logging.INFO, - 2: logging.DEBUG, - }.get(verbosity_level or 0, logging.DEBUG) - - logging.root.setLevel(log_level) - - try: - # This is not listed among the deps because the availability on other platforms is questionable and it's not - # actually required at all. See https://stackoverflow.com/a/16847935/1007777. - import coloredlogs - - # The level spec applies to the handler, not the root logger! This is different from basicConfig(). - coloredlogs.install(level=log_level, fmt=_LOG_FORMAT) - except Exception as ex: - _logger.debug("Colored logs are not available: %s: %s", type(ex), ex) - _logger.info('Consider installing "coloredlogs" from PyPI to make log messages look better') - - # Handle special cases one by one. - if log_level < logging.INFO: - logging.getLogger("pydsdl").setLevel(logging.INFO) # Too much low-level logs from PyDSDL. diff --git a/pyuavcan/_cli/commands/__init__.py b/pyuavcan/_cli/commands/__init__.py deleted file mode 100644 index d900c7ec1..000000000 --- a/pyuavcan/_cli/commands/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import typing -from ._base import Command as Command, SubsystemFactory as SubsystemFactory -from ._paths import DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL as DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL - - -def get_available_command_classes() -> typing.Sequence[typing.Type[Command]]: - import pyuavcan._cli - - # noinspection PyTypeChecker - pyuavcan.util.import_submodules(pyuavcan._cli) - # https://github.com/python/mypy/issues/5374 - return list(pyuavcan.util.iter_descendants(Command)) # type: ignore diff --git a/pyuavcan/_cli/commands/_argparse_helpers.py b/pyuavcan/_cli/commands/_argparse_helpers.py deleted file mode 100644 index 6f854ef25..000000000 --- a/pyuavcan/_cli/commands/_argparse_helpers.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import enum -import typing -import argparse - - -def make_enum_action(enum_type: typing.Type[enum.Enum]) -> typing.Type[argparse.Action]: - mapping: typing.Dict[str, typing.Any] = {} - for e in enum_type: - mapping[e.name.lower()] = e - - class ArgparseEnumAction(argparse.Action): - # noinspection PyShadowingBuiltins - def __init__( - self, - option_strings: typing.Sequence[str], - dest: str, - nargs: typing.Union[int, str, None] = None, - const: typing.Any = None, - default: typing.Any = None, - type: typing.Any = None, - choices: typing.Any = None, - required: bool = False, - help: typing.Optional[str] = None, - metavar: typing.Any = None, - ): - def type_proxy(x: str) -> typing.Any: - """A proxy is needed because a method of an unhashable type is unhashable.""" - return mapping.get(x) - - if type is None: - type = type_proxy - - if choices is None: - choices = [_NamedChoice(key, value) for key, value in mapping.items()] - - super(ArgparseEnumAction, self).__init__( - option_strings, - dest, - nargs=nargs, - const=const, - default=default, - type=type, - choices=choices, - required=required, - help=help, - metavar=metavar, - ) - - def __call__( - self, - parser: argparse.ArgumentParser, - namespace: argparse.Namespace, - values: typing.Union[str, typing.Sequence[typing.Any], None], - option_string: typing.Optional[str] = None, - ) -> None: - setattr(namespace, self.dest, values) - - return ArgparseEnumAction - - -class _NamedChoice: - def __init__(self, key: str, value: typing.Any): - self.key = key - self.value = value - - def __eq__(self, other: object) -> bool: - return bool(self.value == other) - - def __hash__(self) -> int: - return hash(self.value) - - def __repr__(self) -> str: - return self.key diff --git a/pyuavcan/_cli/commands/_base.py b/pyuavcan/_cli/commands/_base.py deleted file mode 100644 index b7929a51c..000000000 --- a/pyuavcan/_cli/commands/_base.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import abc -import typing -import argparse -import pyuavcan -from ._subsystems import SubsystemFactory as SubsystemFactory - - -class Command(abc.ABC): - """ - Base command class. - The constructor shall have no required arguments. - """ - - @property - @abc.abstractmethod - def names(self) -> typing.Sequence[str]: - """ - Command names ordered by preference; first name is the main name. At least one element is required. - """ - raise NotImplementedError - - @property - @abc.abstractmethod - def help(self) -> str: - """ - Documentation help string. Limit the lines to 80 characters max. - """ - raise NotImplementedError - - @property - @abc.abstractmethod - def examples(self) -> typing.Optional[str]: - """ - Set of human-readable usage examples; None if not defined. Limit the lines to 80 characters max. - """ - raise NotImplementedError - - @property - @abc.abstractmethod - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - """ - Subsystems that will be instantiated before the command is executed. - """ - raise NotImplementedError - - @abc.abstractmethod - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - """ - Populates the specified parser instance with command arguments. - """ - raise NotImplementedError - - @abc.abstractmethod - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - """ - Runs the command with the specified arguments and the subsystems constructed from the predefined factories. - """ - raise NotImplementedError - - def __repr__(self) -> str: - return pyuavcan.util.repr_attributes(self, names=self.names) diff --git a/pyuavcan/_cli/commands/_paths.py b/pyuavcan/_cli/commands/_paths.py deleted file mode 100644 index 5d3833776..000000000 --- a/pyuavcan/_cli/commands/_paths.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import os -import sys -import pathlib -import pyuavcan - - -VERSION_AGNOSTIC_DATA_DIR: pathlib.Path -""" -The root directory of version-specific data directories. -Its location is platform-dependent. -It is shared for all versions of the library. -""" - -if hasattr(sys, "getwindowsversion"): # pragma: no cover - _appdata_env = os.getenv("LOCALAPPDATA") or os.getenv("APPDATA") - assert _appdata_env, "Cannot determine the location of the app data directory" - VERSION_AGNOSTIC_DATA_DIR = pathlib.Path(_appdata_env, "UAVCAN", "PyUAVCAN") -else: - VERSION_AGNOSTIC_DATA_DIR = pathlib.Path("~/.uavcan/pyuavcan").expanduser() - -VERSION_SPECIFIC_DATA_DIR: pathlib.Path = VERSION_AGNOSTIC_DATA_DIR / ( - "v" + ".".join(map(str, pyuavcan.__version_info__[:2])) -) -""" -The directory specific to this version of the library where resources and files are stored. -This is always a subdirectory of :data:`VERSION_AGNOSTIC_DATA_DIR`. -The version is specified down to the minor version, ignoring the patch version (e.g, 1.1), -so that versions of the library that differ only by the patch version number will use the same directory. - -This directory contains the default destination path for highly volatile or low-value files. -Having such files segregated by the library version number ensures that when the library is updated, -it will not encounter compatibility issues with older formats. -""" - -OUTPUT_TRANSFER_ID_MAP_DIR: pathlib.Path = VERSION_SPECIFIC_DATA_DIR / "output-transfer-id-maps" -""" -The path is version-specific so that we won't attempt to restore transfer-ID maps stored from another version. -""" - -OUTPUT_TRANSFER_ID_MAP_MAX_AGE = 60.0 # [second] -""" -This is not a path but a related parameter so it's kept here. Files older that this are not used. -""" - -DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL = ( - "https://github.com/UAVCAN/public_regulated_data_types/archive/master.zip" -) diff --git a/pyuavcan/_cli/commands/_subsystems/__init__.py b/pyuavcan/_cli/commands/_subsystems/__init__.py deleted file mode 100644 index cdfe3e898..000000000 --- a/pyuavcan/_cli/commands/_subsystems/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -from ._base import SubsystemFactory as SubsystemFactory - -from . import node as node -from . import formatter as formatter -from . import transport as transport diff --git a/pyuavcan/_cli/commands/_subsystems/_base.py b/pyuavcan/_cli/commands/_subsystems/_base.py deleted file mode 100644 index ee7518f41..000000000 --- a/pyuavcan/_cli/commands/_subsystems/_base.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import abc -import argparse - - -class SubsystemFactory(abc.ABC): - @abc.abstractmethod - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - """ - Populates the provided parser with arguments specific to this subsystem. - """ - raise NotImplementedError - - @abc.abstractmethod - def construct_subsystem(self, args: argparse.Namespace) -> object: - """ - Constructs the product of this factory from the arguments. - """ - raise NotImplementedError diff --git a/pyuavcan/_cli/commands/_subsystems/formatter.py b/pyuavcan/_cli/commands/_subsystems/formatter.py deleted file mode 100644 index c76ea0e55..000000000 --- a/pyuavcan/_cli/commands/_subsystems/formatter.py +++ /dev/null @@ -1,110 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -from __future__ import annotations -import enum -import typing -import logging -import argparse -from .._yaml import YAMLDumper # Reaching to an upper-level module like this is not great, do something about it. -from .._argparse_helpers import make_enum_action -from ._base import SubsystemFactory - - -Formatter = typing.Callable[[typing.Dict[int, typing.Dict[str, typing.Any]]], str] - -_logger = logging.getLogger(__name__) - - -class FormatterFactory(SubsystemFactory): - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - # noinspection PyTypeChecker - parser.add_argument( - "--format", - "-F", - default=next(iter(_Format)), - action=make_enum_action(_Format), - help=""" -The format of the data printed into stdout. The final representation is constructed from an intermediate -"builtin-based" representation, which is a simplified form that is stripped of the detailed DSDL type information, -like JSON. For the background info please read the PyUAVCAN documentation on builtin-based representations. - -YAML is the default option as it is easy to process for humans and other machines alike. Each YAML-formatted object -is separated from its siblings by an explicit document start marker: "---". - -JSON output is optimized for machine parsing, strictly one object per line. - -TSV (tab separated values) output is intended for use with third-party software such as computer algebra systems or -spreadsheet processors. - -Default: %(default)s -""".strip(), - ) - - def construct_subsystem(self, args: argparse.Namespace) -> Formatter: - return { - _Format.YAML: _make_yaml_formatter, - _Format.JSON: _make_json_formatter, - _Format.TSV: _make_tsv_formatter, - }[args.format]() - - -class _Format(enum.Enum): - YAML = enum.auto() - JSON = enum.auto() - TSV = enum.auto() - - -def _make_yaml_formatter() -> Formatter: - dumper = YAMLDumper(explicit_start=True) - return lambda data: dumper.dumps(data) - - -def _make_json_formatter() -> Formatter: - # We prefer simplejson over the standard json because the native json lacks important capabilities: - # - simplejson preserves dict ordering, which is very important for UX. - # - simplejson supports Decimal. - import simplejson as json - - return lambda data: json.dumps(data, ensure_ascii=False, separators=(",", ":")) - - -def _make_tsv_formatter() -> Formatter: - # TODO print into a TSV (tab separated values, like CSV with tabs instead of commas). - # The TSV format should place one scalar per column for ease of parsing by third-party software. - # Variable-length entities such as arrays should expand into the maximum possible number of columns? - # Unions should be represented by adjacent groups of columns where only one such group contains values? - # We may need to obtain the full type information here in order to build the final representation. - # Sounds complex. Search for better ways later. We just need a straightforward way of dumping data into a - # standard tabular format for later processing using third-party software. - raise NotImplementedError("Sorry, the TSV formatter is not yet implemented") - - -def _unittest_formatter() -> None: - obj = { - 2345: { - "abc": { - "def": [ - 123, - 456, - ], - }, - "ghi": 789, - } - } - assert ( - FormatterFactory().construct_subsystem(argparse.Namespace(format=_Format.YAML))(obj) - == """--- -2345: - abc: - def: - - 123 - - 456 - ghi: 789 -""" - ) - assert ( - FormatterFactory().construct_subsystem(argparse.Namespace(format=_Format.JSON))(obj) - == '{"2345":{"abc":{"def":[123,456]},"ghi":789}}' - ) diff --git a/pyuavcan/_cli/commands/_subsystems/node.py b/pyuavcan/_cli/commands/_subsystems/node.py deleted file mode 100644 index b164714bb..000000000 --- a/pyuavcan/_cli/commands/_subsystems/node.py +++ /dev/null @@ -1,226 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import os -import re -import time -import atexit -import pickle -import typing -import logging -import pathlib -import argparse -import pyuavcan -from .._yaml import YAMLLoader, YAMLDumper -from .._paths import OUTPUT_TRANSFER_ID_MAP_DIR, OUTPUT_TRANSFER_ID_MAP_MAX_AGE -from .transport import TransportFactory -from ._base import SubsystemFactory - - -_logger = logging.getLogger(__name__) - - -class NodeFactory(SubsystemFactory): - """ - Constructs a node instance. The instance must be start()ed by the caller afterwards. - """ - - def __init__(self, node_name_suffix: str, allow_anonymous: bool): - self._node_name_suffix = str(node_name_suffix) - self._allow_anonymous = bool(allow_anonymous) - self._transport_factory = TransportFactory() - assert re.match(r"[a-z][a-z0-9_]*[a-z0-9]", self._node_name_suffix), "Poorly chosen name" - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - self._transport_factory.register_arguments(parser) - parser.add_argument( - "--heartbeat-fields", - default="{}", - metavar="YAML_FIELDS", - type=YAMLLoader().load, - help=""" -Value of the heartbeat message uavcan.node.Heartbeat published by the node. The uptime will be overridden so -specifying it here will have no effect. Has no effect if the node is anonymous (i.e., without a local node-ID) because -anonymous nodes do not publish their heartbeat. - -For more info about the YAML representation, read the PyUAVCAN documentation on builtin-based representations. - -Unless overridden, the following defaults are used: -- Mode operational. -- Health nominal. -- Vendor-specific status code equals (PID %% 100) of the command, where PID is its process-ID. -Default: %(default)s -""".strip(), - ) - node_info_fields = { - "protocol_version": { - "major": pyuavcan.UAVCAN_SPECIFICATION_VERSION[0], - "minor": pyuavcan.UAVCAN_SPECIFICATION_VERSION[1], - }, - "software_version": { - "major": pyuavcan.__version_info__[0], - "minor": pyuavcan.__version_info__[1], - }, - "name": "org.uavcan.pyuavcan.cli." + self._node_name_suffix, - } - - def construct_node_info_fields(text: str) -> typing.Dict[str, typing.Any]: - out = node_info_fields.copy() - out.update(YAMLLoader().load(text)) - return out - - parser.add_argument( - "--node-info-fields", - default="{}", - type=construct_node_info_fields, - metavar="YAML_FIELDS", - help=f""" -Value of the node info response uavcan.node.GetInfo returned by the node. This argument overrides the following -defaults per-field: - -{YAMLDumper().dumps(node_info_fields).strip()} - -For more info about the YAML representation, read the PyUAVCAN documentation on builtin-based representations. -Default: %(default)s -""".strip(), - ) - - def construct_subsystem(self, args: argparse.Namespace) -> object: - """ - We use object instead of Node because the Node class requires generated code to be generated. - """ - from pyuavcan import application - from pyuavcan.application import heartbeat_publisher - - node_info = pyuavcan.dsdl.update_from_builtin(application.NodeInfo(), args.node_info_fields) - _logger.debug("Node info: %r", node_info) - - transport = self._transport_factory.construct_subsystem(args) - presentation = pyuavcan.presentation.Presentation(transport) - node = application.Node(presentation, info=node_info) - try: - # Configure the heartbeat publisher. - if args.heartbeat_fields.pop("uptime", None) is not None: - _logger.warning("Specifying uptime has no effect because it will be overridden by the node.") - node.heartbeat_publisher.health = args.heartbeat_fields.pop("health", heartbeat_publisher.Health.NOMINAL) - node.heartbeat_publisher.mode = args.heartbeat_fields.pop("mode", heartbeat_publisher.Mode.OPERATIONAL) - node.heartbeat_publisher.vendor_specific_status_code = args.heartbeat_fields.pop( - "vendor_specific_status_code", os.getpid() % 100 - ) - _logger.debug("Node heartbeat: %r", node.heartbeat_publisher.make_message()) - if args.heartbeat_fields: - raise ValueError(f"Unrecognized heartbeat fields: {args.heartbeat_fields}") - - # Check the node-ID configuration. - if not self._allow_anonymous and node.presentation.transport.local_node_id is None: - raise ValueError( - "The specified transport is configured in anonymous mode, " - "which cannot be used with the selected command. " - "Please specify the node-ID explicitly, or use a different transport." - ) - - # Configure the transfer-ID map. - # Register save on exit even if we're anonymous because the local node-ID may be provided later. - self._register_output_transfer_id_map_save_at_exit(node.presentation) - # Restore if we have a node-ID. If we don't, no restoration will take place even if the node-ID is - # provided later. This behavior is acceptable for CLI; a regular UAVCAN application will not need - # to deal with saving/restoration at all since this use case is specific to CLI only. - path = _get_output_transfer_id_map_path(node.presentation.transport) - tid_map_restored = False - if path is not None: - tid_map = self._restore_output_transfer_id_map(path) - if tid_map: - _logger.debug("Restored output TID map from %s: %r", path, tid_map) - # noinspection PyTypeChecker - presentation.output_transfer_id_map.update(tid_map) # type: ignore - tid_map_restored = True - if not tid_map_restored: - _logger.debug("Could not restore output TID map from %s", path) - - return node - except Exception: - node.close() - raise - - @staticmethod - def _restore_output_transfer_id_map( - file_path: pathlib.Path, - ) -> typing.Dict[object, pyuavcan.presentation.OutgoingTransferIDCounter]: - try: - with open(str(file_path), "rb") as f: - tid_map = pickle.load(f) - except Exception as ex: - _logger.info("Output TID map: Could not restore from file %s: %s: %s", file_path, type(ex).__name__, ex) - return {} - - mtime_abs_diff = abs(file_path.stat().st_mtime - time.time()) - if mtime_abs_diff > OUTPUT_TRANSFER_ID_MAP_MAX_AGE: - _logger.debug( - "Output TID map: File %s is valid but too old: mtime age diff %.0f s", file_path, mtime_abs_diff - ) - return {} - - if isinstance(tid_map, dict) and all( - isinstance(v, pyuavcan.presentation.OutgoingTransferIDCounter) for v in tid_map.values() - ): - return tid_map - else: - _logger.warning( - "Output TID map file %s contains invalid data of type %s", file_path, type(tid_map).__name__ - ) - return {} - - @staticmethod - def _register_output_transfer_id_map_save_at_exit(presentation: pyuavcan.presentation.Presentation) -> None: - # We MUST sample the configuration early because if this is a redundant transport it may reset its - # configuration (local node-ID) back to default after close(). - path = _get_output_transfer_id_map_path(presentation.transport) - _logger.debug("Output TID map file for %s: %s", presentation.transport, path) - - def do_save_at_exit() -> None: - if path is not None: - tmp = f"{path}.{os.getpid()}.{time.time_ns()}.tmp" - _logger.debug("Output TID map save: %s --> %s", tmp, path) - with open(tmp, "wb") as f: - pickle.dump(presentation.output_transfer_id_map, f) - # We use replace for compatibility reasons. On POSIX, a call to rename() will be made, which is - # guaranteed to be atomic. On Windows this may fall back to non-atomic copy, which is still - # acceptable for us here. If the file ends up being damaged, we'll simply ignore it at next startup. - os.replace(tmp, str(path)) - try: - os.unlink(tmp) - except OSError: - pass - - atexit.register(do_save_at_exit) - - -def _get_output_transfer_id_map_path(transport: pyuavcan.transport.Transport) -> typing.Optional[pathlib.Path]: - if transport.local_node_id is not None: - path = OUTPUT_TRANSFER_ID_MAP_DIR / str(transport.local_node_id) - path.parent.mkdir(parents=True, exist_ok=True) - return path - return None - - -def _unittest_output_tid_file_path() -> None: - from pyuavcan.transport.redundant import RedundantTransport - from pyuavcan.transport.loopback import LoopbackTransport - - def once(tr: pyuavcan.transport.Transport) -> typing.Optional[pathlib.Path]: - return _get_output_transfer_id_map_path(tr) - - assert once(LoopbackTransport(None)) is None - assert once(LoopbackTransport(123)) == OUTPUT_TRANSFER_ID_MAP_DIR / "123" - - red = RedundantTransport() - assert once(red) is None - red.attach_inferior(LoopbackTransport(4000)) - red.attach_inferior(LoopbackTransport(4000)) - assert once(red) == OUTPUT_TRANSFER_ID_MAP_DIR / "4000" - - red = RedundantTransport() - red.attach_inferior(LoopbackTransport(None)) - red.attach_inferior(LoopbackTransport(None)) - assert once(red) is None diff --git a/pyuavcan/_cli/commands/_subsystems/transport.py b/pyuavcan/_cli/commands/_subsystems/transport.py deleted file mode 100644 index decba4336..000000000 --- a/pyuavcan/_cli/commands/_subsystems/transport.py +++ /dev/null @@ -1,166 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -from __future__ import annotations -import os -import typing -import inspect -import logging -import argparse -import pyuavcan -from ._base import SubsystemFactory -from .._paths import OUTPUT_TRANSFER_ID_MAP_DIR, OUTPUT_TRANSFER_ID_MAP_MAX_AGE - - -_logger = logging.getLogger(__name__) - - -_ENV_VAR_NAME = "PYUAVCAN_CLI_TRANSPORT" - - -class TransportFactory(SubsystemFactory): - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "--transport", - "--tr", - metavar="EXPRESSION", - action="append", - help=f""" -A Python expression that yields a transport instance upon evaluation. If the expression fails to evaluate or yields -anything that is not a transport instance, the command fails. If the argument is provided more than once, a redundant -transport instance will be constructed automatically. If and only if the transport arguments are not provided, -the transport configuration will be picked up from the environment variable {_ENV_VAR_NAME}. - -Read PyUAVCAN API documentation to learn about the available transports and how their instances can be constructed. - -All nested submodules under "pyuavcan.transport" are imported before the expression is evaluated, so the expression -itself does not need to explicitly import anything. Transports whose dependencies are not installed are silently -skipped; that is, if SerialTransport depends on PySerial but it is not installed, an expression that attempts to -configure a UAVCAN/serial transport would fail to evaluate. - -Examples: - pyuavcan.transport.can.CANTransport(pyuavcan.transport.can.media.socketcan.SocketCANMedia('vcan0', 64), 42) - pyuavcan.transport.loopback.LoopbackTransport(None) - pyuavcan.transport.serial.SerialTransport("/dev/ttyUSB0", None, baudrate=115200) - pyuavcan.transport.udp.UDPTransport('127.42.0.123', anonymous=True) - -Such long expressions are hard to type, so the following entities are also pre-imported into the global namespace -for convenience: - - All direct submodules of "pyuavcan.transport" are wildcard-imported. For example, "pyuavcan.transport.can" - is also available as "can". - - All classes that implement "pyuavcan.transport.Transport" are wildcard-imported under their original name but - without the shared "Transport" suffix. For example, "pyuavcan.transport.loopback.LoopbackTransport" is also - available as "Loopback". -More shortcuts may be added in the future. - -The following examples yield configurations that are equivalent to the above: - CAN(can.media.socketcan.SocketCANMedia('vcan0',64),42) - Loopback(None) - Serial("/dev/ttyUSB0",None,baudrate=115200) - UDP('127.42.0.123',anonymous=True) - -It is often more convenient to use the environment variable instead of typing the arguments because they tend to be -complex and are usually reused without modification. The variable may contain either a single transport expression, -in which case a non-redundant transport instance would be constructed: - {_ENV_VAR_NAME}='Loopback(None)' -...or it may be a Python list/tuple, in which case a redundant transport will be constructed, unless the sequence -contains only one element: - {_ENV_VAR_NAME}="UDP('127.42.0.123'), Serial('/dev/ttyUSB0',None,baudrate=115200)" - -Observe that the node-ID for the local node is to be configured here as well, because per the UAVCAN architecture, -this is a transport-layer property. If desired, a usable node-ID value can be automatically found using the command -"pick-node-id"; read its help for usage information (it's useful for various automation scripts and similar tasks). - -The command-line tool stores the output transfer-ID map on disk keyed by the node-ID; the current local path is: -{OUTPUT_TRANSFER_ID_MAP_DIR} -The map files are managed automatically. They can be removed to reset all transfer-ID counters to zero. Files that -are more than {OUTPUT_TRANSFER_ID_MAP_MAX_AGE} seconds old are no longer used. -""".strip(), - ) - - def construct_subsystem(self, args: argparse.Namespace) -> pyuavcan.transport.Transport: - context = _make_evaluation_context() - trs: typing.List[pyuavcan.transport.Transport] = [] - if args.transport is not None: - _logger.info( - "Configuring the transport from command line arguments; environment variable %s is ignored", - _ENV_VAR_NAME, - ) - for expression in args.transport: - trs += _evaluate_transport_expr(expression, context) - else: - _logger.info( - "Command line arguments do not specify the transport configuration; " - "trying the environment variable %s instead", - _ENV_VAR_NAME, - ) - expression = os.environ.get(_ENV_VAR_NAME, None) - if expression: - trs = _evaluate_transport_expr(expression, context) - - _logger.info("Resulting transport configuration: %r", trs) - if len(trs) < 1: - raise ValueError("No transports specified") - elif len(trs) == 1: - return trs[0] # Non-redundant transport - else: - from pyuavcan.transport.redundant import RedundantTransport - - rt = RedundantTransport() - for t in trs: - rt.attach_inferior(t) - assert rt.inferiors == trs - return rt - - -def _evaluate_transport_expr( - expression: str, context: typing.Dict[str, typing.Any] -) -> typing.List[pyuavcan.transport.Transport]: - out = eval(expression, context) - _logger.debug("Expression %r yields %r", expression, out) - if isinstance(out, pyuavcan.transport.Transport): - return [out] - elif isinstance(out, (list, tuple)) and all(isinstance(x, pyuavcan.transport.Transport) for x in out): - return list(out) - else: - raise ValueError( - f"The expression {expression!r} yields an instance of {type(out).__name__!r}. " - f"Expected an instance of pyuavcan.transport.Transport or a list thereof." - ) - - -def _make_evaluation_context() -> typing.Dict[str, typing.Any]: - def handle_import_error(parent_module_name: str, ex: ImportError) -> None: - try: - tr = parent_module_name.split(".")[2] - except LookupError: - tr = parent_module_name - _logger.info("Transport %r is not available due to the missing dependency %r", tr, ex.name) - - # This import is super slow, so we do it as late as possible. - # Doing this when generating command-line arguments would be disastrous for performance. - # noinspection PyTypeChecker - pyuavcan.util.import_submodules(pyuavcan.transport, error_handler=handle_import_error) - - # Populate the context with all references that may be useful for the transport expression. - context: typing.Dict[str, typing.Any] = { - "pyuavcan": pyuavcan, - } - - # Expose pre-imported transport modules for convenience. - for name, module in inspect.getmembers(pyuavcan.transport, inspect.ismodule): - if not name.startswith("_"): - context[name] = module - - # Pre-import transport classes for convenience. - transport_base = pyuavcan.transport.Transport - # Suppressing MyPy false positive: https://github.com/python/mypy/issues/5374 - for cls in pyuavcan.util.iter_descendants(transport_base): # type: ignore - if not cls.__name__.startswith("_") and cls is not transport_base: - name = cls.__name__.rpartition(transport_base.__name__)[0] - assert name - context[name] = cls - - _logger.debug("Transport expression evaluation context (on the next line):\n%r", context) - return context diff --git a/pyuavcan/_cli/commands/_util.py b/pyuavcan/_cli/commands/_util.py deleted file mode 100644 index 3dcd79146..000000000 --- a/pyuavcan/_cli/commands/_util.py +++ /dev/null @@ -1,137 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import re -import typing -import logging -import decimal -import importlib -import pyuavcan.dsdl -from .dsdl_generate_packages import DSDLGeneratePackagesCommand - - -_NAME_COMPONENT_SEPARATOR = "." - - -_logger = logging.getLogger(__name__) - - -def construct_port_id_and_type(spec: str) -> typing.Tuple[int, typing.Type[pyuavcan.dsdl.CompositeObject]]: - r""" - Parses a data specifier string of the form ``[port_id.]full_data_type_name.major_version.minor_version``. - Name separators may be replaced with ``/`` or ``\`` for compatibility with file system paths; - the version number separators may also be underscores for convenience. - Raises ValueError, possibly with suggestions, if such type is non-reachable. - """ - port_id, full_name, major, minor = _parse_data_spec(spec) - name_components = full_name.split(_NAME_COMPONENT_SEPARATOR) - namespace_components, short_name = name_components[:-1], name_components[-1] - _logger.debug( - "Parsed data spec %r: port_id=%r, namespace_components=%r, short_name=%r, major=%r, minor=%r", - spec, - port_id, - namespace_components, - short_name, - major, - minor, - ) - - # Import the generated data type. - try: - mod = None - for comp in namespace_components: - name = (mod.__name__ + "." + comp) if mod else comp # type: ignore - try: - mod = importlib.import_module(name) - except ImportError: # We seem to have hit a reserved word; try with an underscore. - mod = importlib.import_module(name + "_") - except ImportError: - raise ValueError( - f"The data spec string specifies a non-existent namespace: {spec!r}. " - f"{DSDLGeneratePackagesCommand.make_usage_suggestion_text(namespace_components[0])}" - ) from None - - try: - dtype = getattr(mod, f"{short_name}_{major}_{minor}") - except AttributeError: - raise ValueError(f"The data spec string specifies a non-existent short type name: {spec!r}") from None - - if issubclass(dtype, pyuavcan.dsdl.CompositeObject): - model = pyuavcan.dsdl.get_model(dtype) - port_id = port_id if port_id is not None else model.fixed_port_id - if port_id is None: - raise ValueError( - f"The data spec does not specify a port ID, " - f"and a fixed port ID is not defined for the specified data type: {spec!r}" - ) - return port_id, dtype - else: - raise ValueError(f"The data spec does not specify a valid type: {spec!r}") - - -def convert_transfer_metadata_to_builtin( - transfer: pyuavcan.transport.TransferFrom, **extra_fields: typing.Dict[str, typing.Any] -) -> typing.Dict[str, typing.Any]: - out = { - "timestamp": { - "system": transfer.timestamp.system.quantize(_MICRO), - "monotonic": transfer.timestamp.monotonic.quantize(_MICRO), - }, - "priority": transfer.priority.name.lower(), - "transfer_id": transfer.transfer_id, - "source_node_id": transfer.source_node_id, - } - out.update(extra_fields) - return {"_metadata_": out} - - -_MICRO = decimal.Decimal("0.000001") - -_RE_SPLIT = re.compile(r"^(?:(\d+)\.)?((?:[a-zA-Z_][a-zA-Z0-9_]*)(?:\.[a-zA-Z_][a-zA-Z0-9_]*)*)[_.](\d+)[_.](\d+)$") -""" -Splits ``123.ns.Type.123.45`` into ``('123', 'ns.Type', '123', '45')``. -Splits ``ns.Type.123.45`` into ``(None, 'ns.Type', '123', '45')``. -The version separators (the last two) may be underscores. -""" - - -def _parse_data_spec(spec: str) -> typing.Tuple[typing.Optional[int], str, int, int]: - r""" - Transform the provided data spec into: [port-ID], full name, major version, minor version. - Component separators may be ``/`` or ``\``. Version number separators (the last two) may also be underscores. - Raises ValueError if non-compliant. - """ - spec = spec.strip().replace("/", _NAME_COMPONENT_SEPARATOR).replace("\\", _NAME_COMPONENT_SEPARATOR) - match = _RE_SPLIT.match(spec) - if match is None: - raise ValueError(f"Malformed data spec: {spec!r}") - frag_port_id, frag_full_name, frag_major, frag_minor = match.groups() - return (int(frag_port_id) if frag_port_id is not None else None), frag_full_name, int(frag_major), int(frag_minor) - - -def _unittest_parse_data_spec() -> None: - import pytest - - assert (123, "ns.Type", 12, 34) == _parse_data_spec(" 123.ns.Type.12.34 ") - assert (123, "ns.Type", 12, 34) == _parse_data_spec("123.ns.Type_12.34") - assert (123, "ns.Type", 12, 34) == _parse_data_spec("123.ns/Type.12_34") - assert (123, "ns.Type", 12, 34) == _parse_data_spec("123.ns.Type_12_34") - assert (123, "ns.Type", 12, 34) == _parse_data_spec(r"123\ns\Type_12_34 ") - - assert (None, "ns.Type", 12, 34) == _parse_data_spec("ns.Type.12.34 ") - assert (123, "Type", 12, 34) == _parse_data_spec("123.Type.12.34") - assert (None, "Type", 12, 34) == _parse_data_spec("Type.12.34") - assert (123, "ns0.sub.Type0", 0, 1) == _parse_data_spec("123.ns0.sub.Type0.0.1") - assert (None, "ns0.sub.Type0", 255, 255) == _parse_data_spec(r"ns0/sub\Type0.255.255") - - with pytest.raises(ValueError): - _parse_data_spec("123.ns.Type.12") - with pytest.raises(ValueError): - _parse_data_spec("123.ns.Type.12.43.56") - with pytest.raises(ValueError): - _parse_data_spec("ns.Type.12") - with pytest.raises(ValueError): - _parse_data_spec("ns.Type.12.43.56") - with pytest.raises(ValueError): - _parse_data_spec("123.ns.0Type.12.43") diff --git a/pyuavcan/_cli/commands/_yaml.py b/pyuavcan/_cli/commands/_yaml.py deleted file mode 100644 index 1511dc97e..000000000 --- a/pyuavcan/_cli/commands/_yaml.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -""" -The YAML library we use is API-unstable at the time of writing. We can't just use the de-facto standard PyYAML -because it's kinda stuck in the past (no ordered dicts, no support for YAML v1.2). This facade shields the -rest of the code from breaking changes in the YAML library API or from migration to another library. -""" - -import io -import typing -import decimal -import ruamel.yaml - - -class YAMLDumper: - """ - YAML generation facade. - """ - - def __init__(self, explicit_start: bool = False): - # We need to use the roundtrip representer to retain ordering of mappings, which is important for usability. - self._impl = ruamel.yaml.YAML(typ="rt") - # noinspection PyTypeHints - self._impl.explicit_start = explicit_start # type: ignore - self._impl.default_flow_style = False - - def dump(self, data: typing.Any, stream: typing.TextIO) -> None: - self._impl.dump(data, stream) - - def dumps(self, data: typing.Any) -> str: - s = io.StringIO() - self.dump(data, s) - return s.getvalue() - - -class YAMLLoader: - """ - YAML parsing facade. - Natively represents decimal.Decimal as floats in the output. - """ - - def __init__(self) -> None: - self._impl = ruamel.yaml.YAML() - - def load(self, text: str) -> typing.Any: - return self._impl.load(text) - - -def _represent_decimal(self: ruamel.yaml.BaseRepresenter, data: decimal.Decimal) -> ruamel.yaml.ScalarNode: - if data.is_finite(): - s = str(_POINT_ZERO_DECIMAL + data) # The zero addition is to force float-like string representation - elif data.is_nan(): - s = ".nan" - elif data.is_infinite(): - s = ".inf" if data > 0 else "-.inf" - else: - assert False - return self.represent_scalar("tag:yaml.org,2002:float", s) # type: ignore - - -ruamel.yaml.add_representer(decimal.Decimal, _represent_decimal, representer=ruamel.yaml.RoundTripRepresenter) - -_POINT_ZERO_DECIMAL = decimal.Decimal("0.0") - - -def _unittest_yaml() -> None: - import pytest - - ref = YAMLDumper(explicit_start=True).dumps( - { - "abc": decimal.Decimal("-inf"), - "def": [ - decimal.Decimal("nan"), - { - "qaz": decimal.Decimal("789"), - }, - ], - } - ) - assert ( - ref - == """--- -abc: -.inf -def: -- .nan -- qaz: 789.0 -""" - ) - assert YAMLLoader().load(ref) == { - "abc": -float("inf"), - "def": [ - pytest.approx(float("nan"), nan_ok=True), - { - "qaz": pytest.approx(789), - }, - ], - } diff --git a/pyuavcan/_cli/commands/call.py b/pyuavcan/_cli/commands/call.py deleted file mode 100644 index 9c098de93..000000000 --- a/pyuavcan/_cli/commands/call.py +++ /dev/null @@ -1,232 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import sys -import typing -import asyncio -import decimal -import logging -import argparse -import contextlib -import pyuavcan -from . import _util, _subsystems -from ._argparse_helpers import make_enum_action -from ._yaml import YAMLLoader -from ._base import Command, SubsystemFactory - - -_S = typing.TypeVar("_S", bound=pyuavcan.dsdl.ServiceObject) - - -_logger = logging.getLogger(__name__) - - -class CallCommand(Command): - @property - def names(self) -> typing.Sequence[str]: - return ["call"] - - @property - def help(self) -> str: - return """ -Invoke a service using a specified request object and print the response. The local node will also publish heartbeat -and respond to GetInfo. - -Each emitted output unit is a key-value mapping of one element where the key is the service-ID and the value is the -received response object. The output format is configurable. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return """ -pyuavcan call 42 uavcan.node.GetInfo.1.0 '{}' -""".strip() - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [ - _subsystems.node.NodeFactory(node_name_suffix=self.names[0], allow_anonymous=False), - _subsystems.formatter.FormatterFactory(), - ] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "server_node_id", - metavar="SERVER_NODE_ID", - type=int, - help=f""" -The node ID of the server that the request will be sent to. Valid values range from zero (inclusive) to a -transport-specific upper limit. -""".strip(), - ) - parser.add_argument( - "service_spec", - metavar="[SERVICE_ID.]FULL_SERVICE_TYPE_NAME.MAJOR.MINOR", - help=""" -The full service type name with version and optional service-ID. The service-ID can be omitted if a fixed one is -defined for the data type. - -Forward or backward slashes can be used instead of "."; version numbers can be also separated using underscores. - -Examples: - 123.uavcan.node.ExecuteCommand.1.1 (using service-ID 123) - uavcan/node/ExecuteCommand_1_1 (using the fixed service-ID 435, non-canonical notation) -""".strip(), - ) - parser.add_argument( - "field_spec", - metavar="YAML_FIELDS", - type=YAMLLoader().load, - help=""" -The YAML (or JSON, which is a subset of YAML)-formatted contents of the request object. Missing fields will be left -at their default values. Use empty dict as "{}" to construct a default-initialized request object. For more info about -the YAML representation, read the PyUAVCAN documentation on builtin-based representations. -""".strip(), - ) - parser.add_argument( - "--timeout", - "-T", - metavar="REAL", - type=float, - default=pyuavcan.presentation.DEFAULT_SERVICE_REQUEST_TIMEOUT, - help=f""" -Request timeout; i.e., how long to wait for the response before giving up. -Default: %(default)s -""".strip(), - ) - parser.add_argument( - "--priority", - default=pyuavcan.presentation.DEFAULT_PRIORITY, - action=make_enum_action(pyuavcan.transport.Priority), - help=""" -Priority of the request transfer. Applies to the heartbeat as well. -Default: %(default)s -""".strip(), - ) - parser.add_argument( - "--with-metadata", - "-M", - action="store_true", - help=""" -Emit metadata together with the response. The metadata fields will be contained under the key "_metadata_". -""".strip(), - ) - - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - import pyuavcan.application - - node, formatter = subsystems - assert isinstance(node, pyuavcan.application.Node) - assert callable(formatter) - - with contextlib.closing(node): - node.heartbeat_publisher.priority = args.priority - - # Construct the request object. - service_id, dtype = _util.construct_port_id_and_type(args.service_spec) - if not issubclass(dtype, pyuavcan.dsdl.ServiceObject): - raise ValueError(f"Expected a service type; got this: {dtype.__name__}") - - request = pyuavcan.dsdl.update_from_builtin(dtype.Request(), args.field_spec) - _logger.info("Request object: %r", request) - - # Initialize the client instance. - client = node.presentation.make_client(dtype, service_id, args.server_node_id) - client.response_timeout = args.timeout - client.priority = args.priority - - # Ready to do the job now. - node.start() - return asyncio.get_event_loop().run_until_complete( - _run(client=client, request=request, formatter=formatter, with_metadata=args.with_metadata) - ) - - -async def _run( - client: pyuavcan.presentation.Client[_S], - request: pyuavcan.dsdl.CompositeObject, - formatter: _subsystems.formatter.Formatter, - with_metadata: bool, -) -> int: - request_ts_transport: typing.Optional[pyuavcan.transport.Timestamp] = None - - def on_transfer_feedback(fb: pyuavcan.transport.Feedback) -> None: - nonlocal request_ts_transport - request_ts_transport = fb.first_frame_transmission_timestamp - - client.output_transport_session.enable_feedback(on_transfer_feedback) - - request_ts_application = pyuavcan.transport.Timestamp.now() - result = await client.call(request) - response_ts_application = pyuavcan.transport.Timestamp.now() - - # Print the results. - if result is None: - print(f"The request has timed out after {client.response_timeout:0.1f} seconds", file=sys.stderr) - return 1 - else: - if not request_ts_transport: # pragma: no cover - request_ts_transport = request_ts_application - _logger.error( - "The transport implementation is misbehaving: feedback was never emitted; " - "falling back to software timestamping. " - "Please submit a bug report. Involved instances: client=%r, result=%r", - client, - result, - ) - - response, transfer = result - - transport_duration = transfer.timestamp.monotonic - request_ts_transport.monotonic - application_duration = response_ts_application.monotonic - request_ts_application.monotonic - _logger.info( - "Request duration [second]: " - "transport layer: %.6f, application layer: %.6f, application layer overhead: %.6f", - transport_duration, - application_duration, - application_duration - transport_duration, - ) - - _print_result( - service_id=client.port_id, - response=response, - transfer=transfer, - formatter=formatter, - request_transfer_ts=request_ts_transport, - app_layer_duration=application_duration, - with_metadata=with_metadata, - ) - return 0 - - -def _print_result( - service_id: int, - response: pyuavcan.dsdl.CompositeObject, - transfer: pyuavcan.transport.TransferFrom, - formatter: _subsystems.formatter.Formatter, - request_transfer_ts: pyuavcan.transport.Timestamp, - app_layer_duration: decimal.Decimal, - with_metadata: bool, -) -> None: - bi: typing.Dict[str, typing.Any] = {} # We use updates to ensure proper dict ordering: metadata before data - if with_metadata: - rtt_qnt = decimal.Decimal("0.000001") - bi.update( - _util.convert_transfer_metadata_to_builtin( - transfer, - roundtrip_time={ - "transport_layer": (transfer.timestamp.monotonic - request_transfer_ts.monotonic).quantize(rtt_qnt), - "application_layer": app_layer_duration.quantize(rtt_qnt), - }, - ) - ) - bi.update(pyuavcan.dsdl.to_builtin(response)) - - print( - formatter( - { - service_id: bi, - } - ) - ) diff --git a/pyuavcan/_cli/commands/dsdl_generate_packages.py b/pyuavcan/_cli/commands/dsdl_generate_packages.py deleted file mode 100644 index 39db1c665..000000000 --- a/pyuavcan/_cli/commands/dsdl_generate_packages.py +++ /dev/null @@ -1,208 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import sys -import http -import shutil -import typing -import logging -import pathlib -import zipfile -import tempfile -import argparse -import pyuavcan -from ._base import Command -from ._paths import DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL -from ._subsystems import SubsystemFactory - - -_logger = logging.getLogger(__name__) - - -class DSDLGeneratePackagesCommand(Command): - _SHORT_NAME = "dsdl-gen-pkg" - - @property - def names(self) -> typing.Sequence[str]: - return ["dsdl-generate-packages", self._SHORT_NAME] - - @property - def help(self) -> str: - return """ -Generate PyUAVCAN Python packages from the specified DSDL root namespaces and/or from URLs pointing to an archive -containing a set of DSDL root namespaces. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return f""" -# Generate a package from the root namespace "~/namespace" which depends on public regulated types: -pyuavcan -v dsdl-gen-pkg --lookup {DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL} ~/namespace -""".strip() - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "input", - metavar="INPUT_PATH_OR_URI", - nargs="+", - help=f""" -Either a local path or an URI pointing to the source DSDL root namespace(s). Can be specified more than once to -process multiple namespaces at once. - -If the value is a local path, it must point to a local DSDL root namespace directory or to a local archive containing -DSDL root namespace directories at the top level. If the value is an URI, it must point to an archive containing DSDL -root namespace directories at the top level (this is convenient for generating packages from namespaces hosted in -public repositories, e.g., on GitHub). - -Example path: - ~/uavcan/public_regulated_data_types/uavcan/ - -Example URI: - {DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL} -""".strip(), - ) - parser.add_argument( - "--lookup", - "-L", - action="append", - metavar="LOOKUP_PATH_OR_URI", - help=f""" -This is like --input, except that the specified DSDL root namespace(s) will be used only for looking up dependent -data types; nothing will be generated from these. If a DSDL root namespace is specified as an input, it is -automatically added to the look-up list. - -This option can be specified more than once. -""".strip(), - ) - parser.add_argument( - "--output", - "-O", - help=""" -Path to the directory where the generated packages will be stored. If not specified, defaults to the current working -directory. Existing packages will be overwritten entirely. - -The destination directory should be in the Python module search path list (sys.path or PYTHONPATH) to use the -generated packages. The CLI tool always appends the current working directory to the module search path list -automatically, so in this case the user does not have to do anything manually. -""".strip(), - ) - parser.add_argument( - "--allow-unregulated-fixed-port-id", - action="store_true", - help=""" -Instruct the DSDL front-end to accept unregulated data types with fixed port identifiers. Make sure you understand the -implications before using this option. If not sure, ask for advice at https://forum.uavcan.org. -""".strip(), - ) - - def execute(self, args: argparse.Namespace, _subsystems: typing.Sequence[object]) -> int: - output = pathlib.Path(args.output or pathlib.Path.cwd()) - allow_unregulated_fixed_port_id = bool(args.allow_unregulated_fixed_port_id) - - inputs: typing.List[pathlib.Path] = [] - for location in args.input: - inputs += self._fetch_root_namespace_dirs(location) - _logger.info("Input DSDL root namespace directories: %r", list(map(str, inputs))) - - lookup: typing.List[pathlib.Path] = [] - for location in args.lookup or []: - lookup += self._fetch_root_namespace_dirs(location) - _logger.info("Lookup DSDL root namespace directories: %r", list(map(str, lookup))) - - gpi_list = self._generate_dsdl_packages( - source_root_namespace_dirs=inputs, - lookup_root_namespace_dirs=lookup, - generated_packages_dir=output, - allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, - ) - for gpi in gpi_list: - _logger.info("Generated package %r with %d data types at %r", gpi.name, len(gpi.models), str(gpi.path)) - return 0 - - @staticmethod - def make_usage_suggestion_text(root_namespace_name: str) -> str: - prefix = f"{pathlib.Path(sys.argv[0]).name} {DSDLGeneratePackagesCommand._SHORT_NAME}" - return ( - f'Run "{prefix} DSDL_ROOT_NAMESPACE_PATH_OR_URI" ' - f"to generate the missing Python package from the DSDL namespace {root_namespace_name!r}. " - f'Run "{prefix} --help" for full usage manual.' - ) - - @staticmethod - def _fetch_root_namespace_dirs(location: str) -> typing.List[pathlib.Path]: - if "://" in location: - dirs = DSDLGeneratePackagesCommand._fetch_archive_dirs(location) - _logger.info( - "Resource %r contains the following root namespace directories: %r", location, list(map(str, dirs)) - ) - return dirs - else: - return [pathlib.Path(location)] - - @staticmethod - def _fetch_archive_dirs(archive_url: str) -> typing.List[pathlib.Path]: - """ - Downloads an archive from the specified URL, unpacks it into a temporary directory, and returns the list of - directories in the root of the unpacked archive. - """ - import requests # Takes over 100 ms to import! Having it in the file scope is a performance disaster. - - # TODO: autodetect the type of the archive - arch_dir = tempfile.mkdtemp(prefix="pyuavcan-cli-dsdl") - arch_file = str(pathlib.Path(arch_dir) / "dsdl.zip") - - _logger.info("Downloading the archive from %r into %r...", archive_url, arch_file) - response = requests.get(archive_url) - if response.status_code != http.HTTPStatus.OK: - raise RuntimeError(f"Could not download the archive; HTTP error {response.status_code}") - with open(arch_file, "wb") as f: - f.write(response.content) - - _logger.info("Extracting the archive into %r...", arch_dir) - with zipfile.ZipFile(arch_file) as zf: - zf.extractall(arch_dir) - - (inner,) = [ - d for d in pathlib.Path(arch_dir).iterdir() if d.is_dir() - ] # Strip the outer layer, we don't need it - - assert isinstance(inner, pathlib.Path) - return [d for d in inner.iterdir() if d.is_dir()] - - @staticmethod - def _generate_dsdl_packages( - source_root_namespace_dirs: typing.Iterable[pathlib.Path], - lookup_root_namespace_dirs: typing.Iterable[pathlib.Path], - generated_packages_dir: pathlib.Path, - allow_unregulated_fixed_port_id: bool, - ) -> typing.Sequence[pyuavcan.dsdl.GeneratedPackageInfo]: - lookup_root_namespace_dirs = frozenset(list(lookup_root_namespace_dirs) + list(source_root_namespace_dirs)) - generated_packages_dir.mkdir(parents=True, exist_ok=True) - - out: typing.List[pyuavcan.dsdl.GeneratedPackageInfo] = [] - for ns in source_root_namespace_dirs: - if ns.name.startswith("."): - _logger.debug("Skipping hidden directory %r", ns) - continue - dest_dir = generated_packages_dir / ns.name - _logger.info( - "Generating DSDL package %r from root namespace %r with lookup dirs: %r", - dest_dir, - ns, - list(map(str, lookup_root_namespace_dirs)), - ) - shutil.rmtree(dest_dir, ignore_errors=True) - gpi = pyuavcan.dsdl.generate_package( - root_namespace_directory=ns, - lookup_directories=list(lookup_root_namespace_dirs), - output_directory=generated_packages_dir, - allow_unregulated_fixed_port_id=allow_unregulated_fixed_port_id, - ) - if gpi is not None: - out.append(gpi) - return out diff --git a/pyuavcan/_cli/commands/pick_node_id.py b/pyuavcan/_cli/commands/pick_node_id.py deleted file mode 100644 index 4142dc753..000000000 --- a/pyuavcan/_cli/commands/pick_node_id.py +++ /dev/null @@ -1,130 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import sys -import typing -import random -import asyncio -import logging -import argparse -import contextlib -import pyuavcan -from . import _subsystems -from ._base import Command, SubsystemFactory - - -class PickNodeIDCommand(Command): - @property - def names(self) -> typing.Sequence[str]: - return ["pick-node-id", "pick-nid"] - - @property - def help(self) -> str: - return """ -Automatically find a node-ID value that is not used by any other node that is currently online. This is a simpler -alternative to plug-and-play node-ID allocation logic defined in Specification. Unlike the solution presented there, -this alternative is non-deterministic and collision-prone; it is fundamentally unsafe and it should not be used in -production. Instead, it is intended for use in R&D and testing applications, either directly by humans or from -automation scripts. The operating principle is extremely simple and can be viewed as a simplification of the node-ID -claiming procedure defined in J1939: listen to Heartbeat messages for a short while, build the list of node-ID values -that are currently in use, and then randomly pick a node-ID from the unused ones. The listening duration is determined -heuristically at run time; for most use cases it is unlikely to exceed three seconds. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return None - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [ - _subsystems.transport.TransportFactory(), - ] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - del parser - - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - (transport,) = subsystems - assert isinstance(transport, pyuavcan.transport.Transport) - return asyncio.get_event_loop().run_until_complete(_run(transport=transport)) - - -_logger = logging.getLogger(__name__) - - -async def _run(transport: pyuavcan.transport.Transport) -> int: - import uavcan.node - - if transport.local_node_id is not None: - print("The transport has a valid node-ID already, use it:", transport.local_node_id, file=sys.stderr) - return 2 - - node_id_set_cardinality = transport.protocol_parameters.max_nodes - if node_id_set_cardinality >= 2 ** 32: - # Special case: for very large sets just pick a random number. Very large sets are only possible with test - # transports such as loopback so it's acceptable. If necessary, later we could develop a more robust solution. - print(random.randint(0, node_id_set_cardinality - 1)) - return 0 - - candidates = set(range(node_id_set_cardinality)) - if node_id_set_cardinality > 1000: - # Special case: some transports with large NID cardinality may have difficulties supporting a node-ID of zero - # depending on the configuration of the underlying hardware and software. This is not a problem of UAVCAN but - # of the platform itself. For example, a UDP/IP transport over IPv4 with a node-ID of zero would map to - # an IP address with trailing zeros which happens to be the address of the subnet, which is likely - # to cause all sorts of complications. - _logger.debug("Removing the zero node-ID from the set of available values to avoid platform-specific issues") - candidates.remove(0) - - pres = pyuavcan.presentation.Presentation(transport) - with contextlib.closing(pres): - deadline = asyncio.get_event_loop().time() + uavcan.node.Heartbeat_1_0.MAX_PUBLICATION_PERIOD * 2.0 - sub = pres.make_subscriber_with_fixed_subject_id(uavcan.node.Heartbeat_1_0) - while asyncio.get_event_loop().time() <= deadline: - result = await sub.receive(deadline) - if result is not None: - msg, transfer = result - assert isinstance(transfer, pyuavcan.transport.TransferFrom) - _logger.debug("Received %r via %r", msg, transfer) - if transfer.source_node_id is None: - _logger.warning( - "FYI, the network contains an anonymous node which is publishing Heartbeat. " - "Please contact the vendor and inform them that this behavior is non-compliant. " - "The offending heartbeat message is: %r, transfer: %r", - msg, - transfer, - ) - else: - try: - candidates.remove(int(transfer.source_node_id)) - except LookupError: - pass - else: - # If at least one node is in the Initialization state, the network might be starting, - # so we need to listen longer to minimize the chance of collision. - multiplier = 3.0 if msg.mode.value == uavcan.node.Mode_1_0.INITIALIZATION else 1.0 - advancement = uavcan.node.Heartbeat_1_0.MAX_PUBLICATION_PERIOD * multiplier - _logger.info( - "Deadline advanced by %.1f s; %d candidates left of %d possible", - advancement, - len(candidates), - node_id_set_cardinality, - ) - deadline = max(deadline, asyncio.get_event_loop().time() + advancement) - else: - break - - if not candidates: - print(f"All {node_id_set_cardinality} of the available node-ID values are occupied.", file=sys.stderr) - return 1 - else: - pick = random.choice(list(candidates)) - _logger.info( - "The set of unoccupied node-ID values contains %d elements; the randomly chosen value is %d", - len(candidates), - pick, - ) - print(pick) - return 0 diff --git a/pyuavcan/_cli/commands/publish.py b/pyuavcan/_cli/commands/publish.py deleted file mode 100644 index 98d5960cb..000000000 --- a/pyuavcan/_cli/commands/publish.py +++ /dev/null @@ -1,205 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -from __future__ import annotations -import typing -import asyncio -import logging -import argparse -import contextlib -import pyuavcan -from . import _util, _subsystems -from ._argparse_helpers import make_enum_action -from ._yaml import YAMLLoader -from ._base import Command, SubsystemFactory - - -_logger = logging.getLogger(__name__) - - -class PublishCommand(Command): - @property - def names(self) -> typing.Sequence[str]: - return ["publish", "pub"] - - @property - def help(self) -> str: - return """ -Publish messages of the specified subject with the fixed contents. The local node will also publish heartbeat and -respond to GetInfo, unless it is configured to be anonymous. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return """ -pyuavcan pub uavcan.diagnostic.Record.1.1 '{text: "Hello world!"}' -""".strip() - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [ - _subsystems.node.NodeFactory(node_name_suffix=self.names[0], allow_anonymous=True), - ] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "subject_spec", - metavar="[SUBJECT_ID.]FULL_MESSAGE_TYPE_NAME.MAJOR.MINOR YAML_FIELDS", - nargs="*", - help=""" -The full message type name with version and optional subject-ID, followed by the YAML (or JSON, which is a subset of -YAML)-formatted contents of the message (separated by whitespace). Missing fields will be left at their default values. -Use empty dict as "{}" to construct a default-initialized message. For more info about the YAML representation, read -the PyUAVCAN documentation on builtin-based representations. - -The subject-ID can be omitted if a fixed one is defined for the data type. - -The number of such pairs can be arbitrary; all defined messages will be published synchronously. If no such pairs are -specified, nothing will be published, unless the local node is not anonymous. Per the specification, a non-anonymous -node must publish heartbeat; this requirement is respected. Additionally, the recommended standard service -uavcan.node.GetInfo is served. - -Forward or backward slashes can be used instead of "."; version numbers can be also separated using underscores. - -Examples: - 1234.uavcan.diagnostic.Record.1.1 '{"text": "Hello world!"}' - uavcan/diagnostic/Record_1_1 '{"text": "Hello world!"}' -""".strip(), - ) - parser.add_argument( - "--period", - "-P", - type=float, - default=1.0, - metavar="SECONDS", - help=""" -Message publication period. All messages are published synchronously, so the period setting applies to all specified -subjects. Besides, the period of heartbeat is defined as min((--period), MAX_PUBLICATION_PERIOD); i.e., unless this -value exceeds the maximum period defined for heartbeat by the specification, it is used for heartbeat as well. Note -that anonymous nodes do not publish heartbeat. - -The send timeout for all publishers will equal the publication period. - -Default: %(default)s -""".strip(), - ) - parser.add_argument( - "--count", - "-C", - type=int, - default=1, - metavar="NATURAL", - help=""" -Number of synchronous publication cycles before exiting normally. The duration therefore equals (--period) * (--count). -Default: %(default)s -""".strip(), - ) - parser.add_argument( - "--priority", - default=pyuavcan.presentation.DEFAULT_PRIORITY, - action=make_enum_action(pyuavcan.transport.Priority), - help=""" -Priority of published message transfers. Applies to the heartbeat as well. -Default: %(default)s -""".strip(), - ) - - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - import pyuavcan.application - import pyuavcan.application.heartbeat_publisher - - (node,) = subsystems - assert isinstance(node, pyuavcan.application.Node) - - with contextlib.closing(node): - node.heartbeat_publisher.priority = args.priority - node.heartbeat_publisher.period = min( - pyuavcan.application.heartbeat_publisher.Heartbeat.MAX_PUBLICATION_PERIOD, args.period - ) - - raw_ss = args.subject_spec - if len(raw_ss) % 2 != 0: - raise ValueError( - "Mismatching arguments: each subject specifier must be matched with its field specifier, like: " - "subject-a field-a [subject-b field-b] [...]" - ) - publications: typing.List[Publication] = [] - for subject_spec, field_spec in (raw_ss[i : i + 2] for i in range(0, len(raw_ss), 2)): - publications.append( - Publication( - subject_spec=subject_spec, - field_spec=field_spec, - presentation=node.presentation, - priority=args.priority, - send_timeout=args.period, - ) - ) - _logger.info("Publication set: %r", publications) - - try: - asyncio.get_event_loop().run_until_complete( - self._run(node=node, count=int(args.count), period=float(args.period), publications=publications) - ) - except KeyboardInterrupt: - pass - - if _logger.isEnabledFor(logging.INFO): - _logger.info("%s", node.presentation.transport.sample_statistics()) - for s in node.presentation.transport.output_sessions: - ds = s.specifier.data_specifier - if isinstance(ds, pyuavcan.transport.MessageDataSpecifier): - _logger.info("Subject %d: %s", ds.subject_id, s.sample_statistics()) - - return 0 - - @staticmethod - async def _run(node: object, count: int, period: float, publications: typing.Sequence[Publication]) -> None: - import pyuavcan.application - - assert isinstance(node, pyuavcan.application.Node) - node.start() - - sleep_until = asyncio.get_event_loop().time() - for c in range(count): - out = await asyncio.gather(*[p.publish() for p in publications]) - assert len(out) == len(publications) - assert all(isinstance(x, bool) for x in out) - if not all(out): - log_elements = "\n\t".join(f"#{idx}: {publications[idx]}" for idx, res in enumerate(out) if not res) - _logger.error("The following publications have timed out:\n\t" + log_elements) - - sleep_until += period - _logger.info( - "Publication cycle %d of %d completed; sleeping for %.3f seconds", - c + 1, - count, - sleep_until - asyncio.get_event_loop().time(), - ) - await asyncio.sleep(sleep_until - asyncio.get_event_loop().time()) - - -class Publication: - _YAML_LOADER = YAMLLoader() - - def __init__( - self, - subject_spec: str, - field_spec: str, - presentation: pyuavcan.presentation.Presentation, - priority: pyuavcan.transport.Priority, - send_timeout: float, - ): - subject_id, dtype = _util.construct_port_id_and_type(subject_spec) - content = self._YAML_LOADER.load(field_spec) - - self._message = pyuavcan.dsdl.update_from_builtin(dtype(), content) - self._publisher = presentation.make_publisher(dtype, subject_id) - self._publisher.priority = priority - self._publisher.send_timeout = send_timeout - - async def publish(self) -> bool: - return await self._publisher.publish(self._message) - - def __repr__(self) -> str: - return f"{type(self).__name__}({self._message}, {self._publisher})" diff --git a/pyuavcan/_cli/commands/show_transport.py b/pyuavcan/_cli/commands/show_transport.py deleted file mode 100644 index ba1b489ff..000000000 --- a/pyuavcan/_cli/commands/show_transport.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import typing -import argparse -import pyuavcan -from ._base import Command, SubsystemFactory - - -class ShowTransportCommand(Command): - @property - def names(self) -> typing.Sequence[str]: - return ["show-transport"] - - @property - def help(self) -> str: - return """ -Show transport usage documentation and exit. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return None - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - del parser - - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - import pydoc - - # noinspection PyTypeChecker - pyuavcan.util.import_submodules(pyuavcan.transport) - fill_width = 120 - transport_base = pyuavcan.transport.Transport - # Suppressing MyPy false positive: https://github.com/python/mypy/issues/5374 - for cls in pyuavcan.util.iter_descendants(transport_base): # type: ignore - if not cls.__name__.startswith("_") and cls is not transport_base: - public_module = cls.__module__.split("._")[0] - public_name = public_module + "." + cls.__name__ - print("=" * fill_width) - print(public_name.center(fill_width, " ")) - print("-" * fill_width) - print(cls.__doc__) - print(pydoc.text.document(cls.__init__)) - print() - return 0 diff --git a/pyuavcan/_cli/commands/subscribe.py b/pyuavcan/_cli/commands/subscribe.py deleted file mode 100644 index efd91700a..000000000 --- a/pyuavcan/_cli/commands/subscribe.py +++ /dev/null @@ -1,157 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import typing -import asyncio -import logging -import argparse -import contextlib -import pyuavcan -from . import _util, _subsystems -from ._base import Command, SubsystemFactory - - -_M = typing.TypeVar("_M", bound=pyuavcan.dsdl.CompositeObject) - - -_logger = logging.getLogger(__name__) - - -class SubscribeCommand(Command): - @property - def names(self) -> typing.Sequence[str]: - return ["subscribe", "sub"] - - @property - def help(self) -> str: - return """ -Subscribe to the specified subject, receive and print messages into stdout. This command does not instantiate a local -node; the bus is accessed directly at the presentation layer, so many instances can be cheaply executed concurrently -to subscribe to multiple message streams. - -Each emitted output unit is a key-value mapping where the number of elements equals the number of subjects the command -is asked to subscribe to; the keys are subject-IDs and values are the received message objects. The output format is -configurable. -""".strip() - - @property - def examples(self) -> typing.Optional[str]: - return f""" -pyuavcan sub uavcan.node.Heartbeat.1.0 -""".strip() - - @property - def subsystem_factories(self) -> typing.Sequence[SubsystemFactory]: - return [ - _subsystems.transport.TransportFactory(), - _subsystems.formatter.FormatterFactory(), - ] - - def register_arguments(self, parser: argparse.ArgumentParser) -> None: - parser.add_argument( - "subject_spec", - metavar="[SUBJECT_ID.]FULL_MESSAGE_TYPE_NAME.MAJOR.MINOR", - nargs="+", - help=""" -A set of full message type names with version and optional subject-ID for each. The subject-ID can be omitted if a -fixed one is defined for the data type. If multiple subjects are selected, a synchronizing subscription will be used, -reporting received messages in synchronous groups. - -Forward or backward slashes can be used instead of "."; version numbers can be also separated using underscores. - -Examples: - 1234.uavcan.node.Heartbeat.1.0 (using subject-ID 1234) - uavcan/node/Heartbeat_1_0 (using the fixed subject-ID 7509, non-canonical notation) -""".strip(), - ) - parser.add_argument( - "--with-metadata", - "-M", - action="store_true", - help=""" -Emit metadata together with each message. The metadata fields will be contained under the key "_metadata_". -""".strip(), - ) - parser.add_argument( - "--count", - "-C", - type=int, - metavar="NATURAL", - help=""" -Exit automatically after this many messages (or synchronous message groups) have been received. No limit by default. -""".strip(), - ) - - def execute(self, args: argparse.Namespace, subsystems: typing.Sequence[object]) -> int: - transport, formatter = subsystems - assert isinstance(transport, pyuavcan.transport.Transport) - assert callable(formatter) - - subject_specs = [_util.construct_port_id_and_type(ds) for ds in args.subject_spec] - - _logger.debug( - f"Starting the subscriber with transport={transport}, subject_specs={subject_specs}, " - f"formatter={formatter}, with_metadata={args.with_metadata}" - ) - - with contextlib.closing(pyuavcan.presentation.Presentation(transport)) as presentation: - subscriber = self._make_subscriber(args, presentation) # type: ignore - try: - asyncio.get_event_loop().run_until_complete( - _run( - subscriber=subscriber, - formatter=formatter, - with_metadata=args.with_metadata, - count=int(args.count) if args.count is not None else (2 ** 64), - ) - ) - except KeyboardInterrupt: - pass - - if _logger.isEnabledFor(logging.INFO): - _logger.info("%s", presentation.transport.sample_statistics()) - _logger.info("%s", subscriber.sample_statistics()) - - return 0 - - @staticmethod - def _make_subscriber( - args: argparse.Namespace, presentation: pyuavcan.presentation.Presentation - ) -> pyuavcan.presentation.Subscriber[_M]: - # TODO: the return type will probably have to be changed when multi-subject subscription is supported. - subject_specs = [_util.construct_port_id_and_type(ds) for ds in args.subject_spec] - if len(subject_specs) < 1: - raise ValueError("Nothing to do: no subjects specified") - elif len(subject_specs) == 1: - subject_id, dtype = subject_specs[0] - return presentation.make_subscriber(dtype, subject_id) # type: ignore - else: - raise NotImplementedError( - "Multi-subject subscription is not yet implemented, sorry! " - "See https://github.com/UAVCAN/pyuavcan/issues/65" - ) - - -async def _run( - subscriber: pyuavcan.presentation.Subscriber[_M], - formatter: _subsystems.formatter.Formatter, - with_metadata: bool, - count: int, -) -> None: - async for msg, transfer in subscriber: - assert isinstance(transfer, pyuavcan.transport.TransferFrom) - outer: typing.Dict[int, typing.Dict[str, typing.Any]] = {} - - bi: typing.Dict[str, typing.Any] = {} # We use updates to ensure proper dict ordering: metadata before data - if with_metadata: - bi.update(_util.convert_transfer_metadata_to_builtin(transfer)) - bi.update(pyuavcan.dsdl.to_builtin(msg)) - outer[subscriber.port_id] = bi - - print(formatter(outer)) - - count -= 1 - if count <= 0: - _logger.debug("Reached the specified message count, stopping") - break diff --git a/pyuavcan/application/heartbeat_publisher.py b/pyuavcan/application/heartbeat_publisher.py index 86e0d8b4d..d75a543ee 100644 --- a/pyuavcan/application/heartbeat_publisher.py +++ b/pyuavcan/application/heartbeat_publisher.py @@ -74,10 +74,8 @@ def __init__(self, presentation: pyuavcan.presentation.Presentation): self._vendor_specific_status_code = 0 self._pre_heartbeat_handlers: typing.List[typing.Callable[[], None]] = [] self._maybe_task: typing.Optional[asyncio.Task[None]] = None - - self._publisher = self._presentation.make_publisher_with_fixed_subject_id(Heartbeat) - self._publisher.send_timeout = float(Heartbeat.MAX_PUBLICATION_PERIOD) - + self._priority = pyuavcan.presentation.DEFAULT_PRIORITY + self._period = float(Heartbeat.MAX_PUBLICATION_PERIOD) self._subscriber = self._presentation.make_subscriber_with_fixed_subject_id(Heartbeat) def start(self) -> None: @@ -132,15 +130,14 @@ def period(self) -> float: """ How often the Heartbeat messages should be published. The upper limit (i.e., the lowest frequency) is constrained by the UAVCAN specification; please see the DSDL source of ``uavcan.node.Heartbeat``. - The send timeout equals the period. """ - return self._publisher.send_timeout + return self._period @period.setter def period(self, value: float) -> None: value = float(value) if 0 < value <= Heartbeat.MAX_PUBLICATION_PERIOD: - self._publisher.send_timeout = value # This is not a typo! Send timeout equals period here. + self._period = value else: raise ValueError(f"Invalid heartbeat period: {value}") @@ -149,18 +146,12 @@ def priority(self) -> pyuavcan.transport.Priority: """ The transfer priority level to use when publishing Heartbeat messages. """ - return self._publisher.priority + return self._priority @priority.setter def priority(self, value: pyuavcan.transport.Priority) -> None: - self._publisher.priority = pyuavcan.transport.Priority(value) - - @property - def publisher(self) -> pyuavcan.presentation.Publisher[Heartbeat]: - """ - Provides access to the underlying presentation layer publisher instance (see constructor). - """ - return self._publisher + # noinspection PyArgumentList + self._priority = pyuavcan.transport.Priority(value) def add_pre_heartbeat_handler(self, handler: typing.Callable[[], None]) -> None: """ @@ -195,36 +186,35 @@ def close(self) -> None: """ if self._maybe_task: self._subscriber.close() - self._publisher.close() self._maybe_task.cancel() self._maybe_task = None async def _task_function(self) -> None: next_heartbeat_at = time.monotonic() - while self._maybe_task: - try: - self._call_pre_heartbeat_handlers() - if self._presentation.transport.local_node_id is not None: - if not await self._publisher.publish(self.make_message()): - _logger.warning("%s heartbeat send timed out", self) - - next_heartbeat_at += self._publisher.send_timeout - await asyncio.sleep(next_heartbeat_at - time.monotonic()) - except asyncio.CancelledError: - _logger.debug("%s publisher task cancelled", self) - break - except pyuavcan.transport.ResourceClosedError as ex: - _logger.debug("%s transport closed, publisher task will exit: %s", self, ex) - break - except Exception as ex: - _logger.exception("%s publisher task exception: %s", self, ex) + pub: typing.Optional[pyuavcan.presentation.Publisher[Heartbeat]] = None try: - self._publisher.close() - except pyuavcan.transport.TransportError: - pass - - def _call_pre_heartbeat_handlers(self) -> None: - pyuavcan.util.broadcast(self._pre_heartbeat_handlers)() + while self._maybe_task: + try: + pyuavcan.util.broadcast(self._pre_heartbeat_handlers)() + if self._presentation.transport.local_node_id is not None: + if pub is None: + pub = self._presentation.make_publisher_with_fixed_subject_id(Heartbeat) + assert pub is not None + pub.priority = self._priority + if not await pub.publish(self.make_message()): + _logger.error("%s heartbeat send timed out", self) + except (asyncio.CancelledError, pyuavcan.transport.ResourceClosedError) as ex: + _logger.debug("%s publisher task will exit: %s", self, ex) + break + except Exception as ex: # pragma: no cover + _logger.exception("%s publisher task exception: %s", self, ex) + + next_heartbeat_at += self._period + await asyncio.sleep(next_heartbeat_at - time.monotonic()) + finally: + _logger.debug("%s publisher task is stopping", self) + if pub is not None: + pub.close() async def _handle_received_heartbeat(self, msg: Heartbeat, metadata: pyuavcan.transport.TransferFrom) -> None: local_node_id = self._presentation.transport.local_node_id @@ -239,4 +229,9 @@ async def _handle_received_heartbeat(self, msg: Heartbeat, metadata: pyuavcan.tr ) def __repr__(self) -> str: - return pyuavcan.util.repr_attributes(self, heartbeat=self.make_message(), publisher=self._publisher) + return pyuavcan.util.repr_attributes( + self, + heartbeat=self.make_message(), + priority=self._priority.name, + period=self._period, + ) diff --git a/pyuavcan/dsdl/_builtin_form.py b/pyuavcan/dsdl/_builtin_form.py index c5ca757ab..519d77afc 100644 --- a/pyuavcan/dsdl/_builtin_form.py +++ b/pyuavcan/dsdl/_builtin_form.py @@ -80,9 +80,7 @@ def _to_builtin_impl( assert False, "Unexpected inputs" -def update_from_builtin( - destination: CompositeObjectTypeVar, source: typing.Dict[str, typing.Any] -) -> CompositeObjectTypeVar: +def update_from_builtin(destination: CompositeObjectTypeVar, source: typing.Any) -> CompositeObjectTypeVar: """ Updates the provided DSDL object (an instance of a Python class auto-generated from a DSDL definition) with the values from a native representation, where DSDL objects are represented as dicts, arrays diff --git a/pyuavcan/dsdl/_compiler.py b/pyuavcan/dsdl/_compiler.py index c98d52bd9..bee42e4d5 100644 --- a/pyuavcan/dsdl/_compiler.py +++ b/pyuavcan/dsdl/_compiler.py @@ -181,6 +181,10 @@ def generate_package( """ started_at = time.monotonic() + if isinstance(lookup_directories, (str, bytes, pathlib.Path)): + # https://forum.uavcan.org/t/nestedrootnamespaceerror-in-basic-usage-demo/794 + raise TypeError(f"Lookup directories shall be an iterable of paths, not {type(lookup_directories).__name__}") + output_directory = pathlib.Path(pathlib.Path.cwd() if output_directory is None else output_directory).resolve() root_namespace_directory = pathlib.Path(root_namespace_directory).resolve() if root_namespace_directory.parent == output_directory: @@ -191,9 +195,6 @@ def generate_package( ) # Read the DSDL definitions - if isinstance(lookup_directories, (str, bytes)): - # https://forum.uavcan.org/t/nestedrootnamespaceerror-in-basic-usage-demo/794 - raise TypeError(f"Lookup directories shall be an iterable of strings, not {type(lookup_directories).__name__}") composite_types = pydsdl.read_namespace( root_namespace_directory=str(root_namespace_directory), lookup_directories=list(map(str, lookup_directories or [])), @@ -251,7 +252,7 @@ def generate_package( quick_fix = f'Quick fix: `export PYTHONPATH="{output_directory.resolve()}"`' else: quick_fix = "Quick fix is not available for this OS." - _logger.warning( + _logger.info( "Generated package is stored in %r, which is not in Python module search path list. " "The package will fail to import unless you add the destination directory to sys.path or PYTHONPATH. %s", str(output_directory), diff --git a/pyuavcan/presentation/_port/_client.py b/pyuavcan/presentation/_port/_client.py index 1efdde57a..18a3afae9 100644 --- a/pyuavcan/presentation/_port/_client.py +++ b/pyuavcan/presentation/_port/_client.py @@ -219,7 +219,7 @@ def __init__( self._maybe_finalizer: typing.Optional[PortFinalizer] = finalizer self._loop = loop - self._lock = asyncio.Lock(loop=loop) + self._lock = asyncio.Lock() self._proxy_count = 0 self._response_futures_by_transfer_id: typing.Dict[ int, asyncio.Future[typing.Tuple[pyuavcan.dsdl.CompositeObject, pyuavcan.transport.TransferFrom]] @@ -265,7 +265,7 @@ async def call( try: if send_result: self.sent_request_count += 1 - response, transfer = await asyncio.wait_for(future, timeout=response_timeout, loop=self._loop) + response, transfer = await asyncio.wait_for(future, timeout=response_timeout) assert isinstance(response, self.dtype.Response) assert isinstance(transfer, pyuavcan.transport.TransferFrom) return response, transfer diff --git a/pyuavcan/presentation/_port/_publisher.py b/pyuavcan/presentation/_port/_publisher.py index 0b1583e89..d6107acf9 100644 --- a/pyuavcan/presentation/_port/_publisher.py +++ b/pyuavcan/presentation/_port/_publisher.py @@ -166,7 +166,7 @@ def __init__( self.transfer_id_counter = transfer_id_counter self._maybe_finalizer: typing.Optional[PortFinalizer] = finalizer self._loop = loop - self._lock = asyncio.Lock(loop=loop) + self._lock = asyncio.Lock() self._proxy_count = 0 async def publish( diff --git a/pyuavcan/presentation/_port/_server.py b/pyuavcan/presentation/_port/_server.py index cdf008816..e2a3acfb4 100644 --- a/pyuavcan/presentation/_port/_server.py +++ b/pyuavcan/presentation/_port/_server.py @@ -65,7 +65,7 @@ class ServiceRequestMetadata: def __repr__(self) -> str: kwargs = {f.name: getattr(self, f.name) for f in dataclasses.fields(self)} - kwargs["priority"] = str(self.priority).split(".")[-1] + kwargs["priority"] = self.priority.name del kwargs["timestamp"] return pyuavcan.util.repr_attributes(self, str(self.timestamp), **kwargs) diff --git a/pyuavcan/presentation/_port/_subscriber.py b/pyuavcan/presentation/_port/_subscriber.py index bb00270ce..b64e705b1 100644 --- a/pyuavcan/presentation/_port/_subscriber.py +++ b/pyuavcan/presentation/_port/_subscriber.py @@ -75,7 +75,7 @@ def __init__( self._impl = impl self._loop = loop self._maybe_task: typing.Optional[asyncio.Task[None]] = None - self._rx: _Listener[MessageClass] = _Listener(asyncio.Queue(maxsize=queue_capacity, loop=loop)) + self._rx: _Listener[MessageClass] = _Listener(asyncio.Queue(maxsize=queue_capacity)) impl.add_listener(self._rx) # ---------------------------------------- HANDLER-BASED API ---------------------------------------- @@ -151,7 +151,7 @@ async def receive_for( self._raise_if_closed_or_failed() try: if timeout > 0: - message, transfer = await asyncio.wait_for(self._rx.queue.get(), timeout, loop=self._loop) + message, transfer = await asyncio.wait_for(self._rx.queue.get(), timeout) else: message, transfer = self._rx.queue.get_nowait() except asyncio.QueueEmpty: diff --git a/pyuavcan/transport/_tracer.py b/pyuavcan/transport/_tracer.py index 301bf4adc..9940a6f43 100644 --- a/pyuavcan/transport/_tracer.py +++ b/pyuavcan/transport/_tracer.py @@ -60,7 +60,7 @@ class AlienTransferMetadata: def __repr__(self) -> str: return pyuavcan.util.repr_attributes( - self, self.session_specifier, priority=str(self.priority).split(".")[-1], transfer_id=self.transfer_id + self, self.session_specifier, priority=self.priority.name, transfer_id=self.transfer_id ) diff --git a/pyuavcan/transport/_transfer.py b/pyuavcan/transport/_transfer.py index dd7231037..86b19a775 100644 --- a/pyuavcan/transport/_transfer.py +++ b/pyuavcan/transport/_transfer.py @@ -71,7 +71,7 @@ class Transfer: def __repr__(self) -> str: fragmented_payload = "+".join(f"{len(x)}B" for x in self.fragmented_payload) kwargs = {f.name: getattr(self, f.name) for f in dataclasses.fields(self)} - kwargs["priority"] = str(self.priority).split(".")[-1] + kwargs["priority"] = self.priority.name kwargs["fragmented_payload"] = f"[{fragmented_payload}]" del kwargs["timestamp"] return pyuavcan.util.repr_attributes(self, str(self.timestamp), **kwargs) diff --git a/pyuavcan/transport/can/_can.py b/pyuavcan/transport/can/_can.py index 1444ba74d..eb083e623 100644 --- a/pyuavcan/transport/can/_can.py +++ b/pyuavcan/transport/can/_can.py @@ -83,7 +83,7 @@ def __init__( """ self._maybe_media: typing.Optional[Media] = media self._local_node_id = int(local_node_id) if local_node_id is not None else None - self._media_lock = asyncio.Lock(loop=loop) + self._media_lock = asyncio.Lock() self._loop = loop if loop is not None else asyncio.get_event_loop() # Lookup performance for the output registry is not important because it's only used for loopback frames. diff --git a/pyuavcan/transport/can/_session/_input.py b/pyuavcan/transport/can/_session/_input.py index b55b9d548..e24b7ed50 100644 --- a/pyuavcan/transport/can/_session/_input.py +++ b/pyuavcan/transport/can/_session/_input.py @@ -94,7 +94,7 @@ def frame_queue_capacity(self, value: typing.Optional[int]) -> None: raise ValueError(f"Invalid value for queue capacity: {value}") old_queue = self._queue - self._queue = asyncio.Queue(int(value) if value is not None else 0, loop=self._loop) + self._queue = asyncio.Queue(int(value) if value is not None else 0) try: while True: self._push_frame(*old_queue.get_nowait()) @@ -139,7 +139,7 @@ async def _do_receive(self, monotonic_deadline: float) -> typing.Optional[pyuavc # Continue reading past the deadline until the queue is empty or a transfer is received. timeout = monotonic_deadline - self._loop.time() if timeout > 0: - timestamp, canid, frame = await asyncio.wait_for(self._queue.get(), timeout, loop=self._loop) + timestamp, canid, frame = await asyncio.wait_for(self._queue.get(), timeout) else: timestamp, canid, frame = self._queue.get_nowait() assert isinstance(timestamp, Timestamp) diff --git a/pyuavcan/transport/can/_session/_output.py b/pyuavcan/transport/can/_session/_output.py index ba68d2380..7e6cee7ee 100644 --- a/pyuavcan/transport/can/_session/_output.py +++ b/pyuavcan/transport/can/_session/_output.py @@ -221,6 +221,10 @@ def __init__( raise pyuavcan.transport.UnsupportedSessionConfigurationError( f"This transport does not support unicast outputs for {specifier.data_specifier}" ) + if transport.local_node_id is None: + raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( + "Cannot emit service transfers because the local node is anonymous (does not have a node-ID)" + ) self._service_id = specifier.data_specifier.service_id self._request_not_response = ( specifier.data_specifier.role == pyuavcan.transport.ServiceDataSpecifier.Role.REQUEST @@ -236,11 +240,7 @@ def __init__( async def send(self, transfer: pyuavcan.transport.Transfer, monotonic_deadline: float) -> bool: source_node_id = self._transport.local_node_id - if source_node_id is None: - raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( - "Cannot emit a service transfer because the local node is anonymous (does not have a node-ID)" - ) - + assert source_node_id is not None, "Internal logic error" can_id = ServiceCANID( priority=transfer.priority, service_id=self._service_id, diff --git a/pyuavcan/transport/can/media/_frame.py b/pyuavcan/transport/can/media/_frame.py index 427cbe00c..1fa58ebb8 100644 --- a/pyuavcan/transport/can/media/_frame.py +++ b/pyuavcan/transport/can/media/_frame.py @@ -59,9 +59,7 @@ def __repr__(self) -> str: FrameFormat.EXTENDED: "0x%08x", FrameFormat.BASE: "0x%03x", }[self.format] % self.identifier - return pyuavcan.util.repr_attributes( - self, format=str(self.format).split(".")[-1], identifier=ide, data=self.data.hex() - ) + return pyuavcan.util.repr_attributes(self, format=self.format.name, identifier=ide, data=self.data.hex()) @dataclasses.dataclass(frozen=True) diff --git a/pyuavcan/transport/can/media/socketcan/_socketcan.py b/pyuavcan/transport/can/media/socketcan/_socketcan.py index 56e1b5c0b..3ec197ce4 100644 --- a/pyuavcan/transport/can/media/socketcan/_socketcan.py +++ b/pyuavcan/transport/can/media/socketcan/_socketcan.py @@ -124,7 +124,6 @@ async def send(self, frames: typing.Iterable[Envelope], monotonic_deadline: floa await asyncio.wait_for( self._loop.sock_sendall(self._sock, self._compile_native_frame(f.frame)), timeout=monotonic_deadline - self._loop.time(), - loop=self._loop, ) except asyncio.TimeoutError: break diff --git a/pyuavcan/transport/commons/high_overhead_transport/_frame.py b/pyuavcan/transport/commons/high_overhead_transport/_frame.py index 22aaaf487..428d06d35 100644 --- a/pyuavcan/transport/commons/high_overhead_transport/_frame.py +++ b/pyuavcan/transport/commons/high_overhead_transport/_frame.py @@ -73,7 +73,7 @@ def __repr__(self) -> str: else: payload = bytes(self.payload).hex() kwargs = {f.name: getattr(self, f.name) for f in dataclasses.fields(self)} - kwargs["priority"] = str(self.priority).split(".")[-1] + kwargs["priority"] = self.priority.name kwargs["payload"] = payload return pyuavcan.util.repr_attributes(self, **kwargs) diff --git a/pyuavcan/transport/loopback/_input_session.py b/pyuavcan/transport/loopback/_input_session.py index a3a2b6db4..d352e07a0 100644 --- a/pyuavcan/transport/loopback/_input_session.py +++ b/pyuavcan/transport/loopback/_input_session.py @@ -24,14 +24,14 @@ def __init__( self._closer = closer self._transfer_id_timeout = float(self.DEFAULT_TRANSFER_ID_TIMEOUT) self._stats = pyuavcan.transport.SessionStatistics() - self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue(loop=loop) + self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue() super(LoopbackInputSession, self).__init__() async def receive(self, monotonic_deadline: float) -> typing.Optional[pyuavcan.transport.TransferFrom]: timeout = monotonic_deadline - self._loop.time() try: if timeout > 0: - out = await asyncio.wait_for(self._queue.get(), timeout, loop=self._loop) + out = await asyncio.wait_for(self._queue.get(), timeout) else: out = self._queue.get_nowait() except asyncio.TimeoutError: diff --git a/pyuavcan/transport/loopback/_loopback.py b/pyuavcan/transport/loopback/_loopback.py index 568bd5f56..74b8f3750 100644 --- a/pyuavcan/transport/loopback/_loopback.py +++ b/pyuavcan/transport/loopback/_loopback.py @@ -26,9 +26,16 @@ class LoopbackTransport(pyuavcan.transport.Transport): The only valid usage is sending and receiving same data on the same node. """ - def __init__(self, local_node_id: typing.Optional[int], *, loop: typing.Optional[asyncio.AbstractEventLoop] = None): + def __init__( + self, + local_node_id: typing.Optional[int], + *, + allow_anonymous_transfers: bool = True, + loop: typing.Optional[asyncio.AbstractEventLoop] = None, + ): self._loop = loop if loop is not None else asyncio.get_event_loop() self._local_node_id = int(local_node_id) if local_node_id is not None else None + self._allow_anonymous_transfers = allow_anonymous_transfers self._input_sessions: typing.Dict[pyuavcan.transport.InputSessionSpecifier, LoopbackInputSession] = {} self._output_sessions: typing.Dict[pyuavcan.transport.OutputSessionSpecifier, LoopbackOutputSession] = {} self._capture_handlers: typing.List[pyuavcan.transport.CaptureCallback] = [] @@ -132,6 +139,10 @@ async def do_route(tr: pyuavcan.transport.Transfer, monotonic_deadline: float) - try: sess = self._output_sessions[specifier] except KeyError: + if self.local_node_id is None and not self._allow_anonymous_transfers: + raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( + f"Anonymous transfers are not enabled for {self}" + ) sess = LoopbackOutputSession( specifier=specifier, payload_metadata=payload_metadata, loop=self.loop, closer=do_close, router=do_route ) @@ -169,4 +180,5 @@ def capture_handlers(self) -> typing.Sequence[pyuavcan.transport.CaptureCallback def _get_repr_fields(self) -> typing.Tuple[typing.List[typing.Any], typing.Dict[str, typing.Any]]: return [], { "local_node_id": self.local_node_id, + "allow_anonymous_transfers": self._allow_anonymous_transfers, } diff --git a/pyuavcan/transport/redundant/_session/_input.py b/pyuavcan/transport/redundant/_session/_input.py index 3d1d4ce2c..5d73aa009 100644 --- a/pyuavcan/transport/redundant/_session/_input.py +++ b/pyuavcan/transport/redundant/_session/_input.py @@ -74,9 +74,9 @@ def __init__( self._deduplicator: typing.Optional[Deduplicator] = None # The actual deduplicated transfers received by the inferiors. - self._read_queue: asyncio.Queue[RedundantTransferFrom] = asyncio.Queue(loop=loop) + self._read_queue: asyncio.Queue[RedundantTransferFrom] = asyncio.Queue() # Queuing errors is meaningless because they lose relevance immediately, so the queue is only one item deep. - self._error_queue: asyncio.Queue[Exception] = asyncio.Queue(1, loop=loop) + self._error_queue: asyncio.Queue[Exception] = asyncio.Queue(1) self._stat_transfers = 0 self._stat_payload_bytes = 0 @@ -156,7 +156,7 @@ async def receive(self, monotonic_deadline: float) -> typing.Optional[RedundantT try: timeout = monotonic_deadline - self._loop.time() if timeout > 0: - tr = await asyncio.wait_for(self._read_queue.get(), timeout, loop=self._loop) + tr = await asyncio.wait_for(self._read_queue.get(), timeout) else: tr = self._read_queue.get_nowait() except (asyncio.TimeoutError, asyncio.QueueEmpty): diff --git a/pyuavcan/transport/redundant/_session/_output.py b/pyuavcan/transport/redundant/_session/_output.py index 0542708da..5826e8be5 100644 --- a/pyuavcan/transport/redundant/_session/_output.py +++ b/pyuavcan/transport/redundant/_session/_output.py @@ -83,7 +83,7 @@ def __init__( self._inferiors: typing.List[pyuavcan.transport.OutputSession] = [] self._feedback_handler: typing.Optional[typing.Callable[[RedundantFeedback], None]] = None self._idle_send_future: typing.Optional[asyncio.Future[None]] = None - self._lock = asyncio.Lock(loop=self._loop) + self._lock = asyncio.Lock() self._stat_transfers = 0 self._stat_payload_bytes = 0 @@ -185,9 +185,7 @@ async def send(self, transfer: pyuavcan.transport.Transfer, monotonic_deadline: _logger.debug("%s has no inferiors; suspending the send method...", self) self._idle_send_future = self._loop.create_future() try: - await asyncio.wait_for( - self._idle_send_future, timeout=monotonic_deadline - self._loop.time(), loop=self._loop - ) + await asyncio.wait_for(self._idle_send_future, timeout=monotonic_deadline - self._loop.time()) except asyncio.TimeoutError: pass else: diff --git a/pyuavcan/transport/serial/_serial.py b/pyuavcan/transport/serial/_serial.py index 68ad0122e..68387aee8 100644 --- a/pyuavcan/transport/serial/_serial.py +++ b/pyuavcan/transport/serial/_serial.py @@ -127,7 +127,7 @@ def __init__( self._closed = False # For serial port write serialization. Read operations are performed concurrently (no sync) in separate thread. - self._port_lock = asyncio.Lock(loop=loop) + self._port_lock = asyncio.Lock() # The serialization buffer is re-used for performance reasons; it is needed to store frame contents before # they are emitted into the serial port. It may grow as necessary at runtime; the initial size is a guess. @@ -220,7 +220,6 @@ def finalizer() -> None: async def send_transfer( frames: typing.List[SerialFrame], monotonic_deadline: float ) -> typing.Optional[Timestamp]: - frames = list(frames) first_tx_ts: typing.Optional[Timestamp] = None for _ in range(self._service_transfer_multiplier): # pragma: no branch ts = await self._send_transfer(frames, monotonic_deadline) diff --git a/pyuavcan/transport/serial/_session/_input.py b/pyuavcan/transport/serial/_session/_input.py index ffad86d25..4afd9753b 100644 --- a/pyuavcan/transport/serial/_session/_input.py +++ b/pyuavcan/transport/serial/_session/_input.py @@ -50,14 +50,9 @@ def __init__( self._loop = loop assert self._loop is not None - if not isinstance(self._specifier, pyuavcan.transport.InputSessionSpecifier) or not isinstance( - self._payload_metadata, pyuavcan.transport.PayloadMetadata - ): # pragma: no cover - raise TypeError("Invalid parameters") - self._statistics = SerialInputSessionStatistics() self._transfer_id_timeout = self.DEFAULT_TRANSFER_ID_TIMEOUT - self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue(loop=loop) + self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue() self._reassemblers: typing.Dict[int, TransferReassembler] = {} super(SerialInputSession, self).__init__(finalizer) @@ -95,7 +90,7 @@ async def receive(self, monotonic_deadline: float) -> typing.Optional[pyuavcan.t try: timeout = monotonic_deadline - self._loop.time() if timeout > 0: - transfer = await asyncio.wait_for(self._queue.get(), timeout, loop=self._loop) + transfer = await asyncio.wait_for(self._queue.get(), timeout) else: transfer = self._queue.get_nowait() except (asyncio.TimeoutError, asyncio.QueueEmpty): diff --git a/pyuavcan/transport/serial/_session/_output.py b/pyuavcan/transport/serial/_session/_output.py index 6d5e8ff35..e60cbf535 100644 --- a/pyuavcan/transport/serial/_session/_output.py +++ b/pyuavcan/transport/serial/_session/_output.py @@ -7,6 +7,7 @@ import typing import logging import pyuavcan +from pyuavcan.transport import ServiceDataSpecifier from .._frame import SerialFrame from ._base import SerialSession @@ -58,18 +59,14 @@ def __init__( self._send_handler = send_handler self._feedback_handler: typing.Optional[typing.Callable[[pyuavcan.transport.Feedback], None]] = None self._statistics = pyuavcan.transport.SessionStatistics() + if self._local_node_id is None and isinstance(self._specifier.data_specifier, ServiceDataSpecifier): + raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( + f"Anonymous nodes cannot emit service transfers. Session specifier: {self._specifier}" + ) assert isinstance(self._local_node_id, int) or self._local_node_id is None assert callable(send_handler) - - if not isinstance(self._specifier, pyuavcan.transport.OutputSessionSpecifier) or not isinstance( - self._payload_metadata, pyuavcan.transport.PayloadMetadata - ): # pragma: no cover - raise TypeError("Invalid parameters") - assert ( - specifier.remote_node_id is not None - if isinstance(specifier.data_specifier, pyuavcan.transport.ServiceDataSpecifier) - else True + specifier.remote_node_id is not None if isinstance(specifier.data_specifier, ServiceDataSpecifier) else True ), "Internal protocol violation: cannot broadcast a service transfer" super(SerialOutputSession, self).__init__(finalizer) @@ -77,13 +74,6 @@ def __init__( async def send(self, transfer: pyuavcan.transport.Transfer, monotonic_deadline: float) -> bool: self._raise_if_closed() - if self._local_node_id is None and isinstance( - self._specifier.data_specifier, pyuavcan.transport.ServiceDataSpecifier - ): - raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( - f"Anonymous nodes cannot emit service transfers. Session specifier: {self._specifier}" - ) - def construct_frame(index: int, end_of_transfer: bool, payload: memoryview) -> SerialFrame: if not end_of_transfer and self._local_node_id is None: raise pyuavcan.transport.OperationNotDefinedForAnonymousNodeError( @@ -179,21 +169,14 @@ def do_finalize() -> None: nonlocal finalized finalized = True - sos = SerialOutputSession( - specifier=OutputSessionSpecifier(ServiceDataSpecifier(321, ServiceDataSpecifier.Role.REQUEST), 1111), - payload_metadata=PayloadMetadata(1024), - mtu=10, - local_node_id=None, # pragma: no cover - send_handler=do_send, - finalizer=do_finalize, - ) - with raises(pyuavcan.transport.OperationNotDefinedForAnonymousNodeError): - run_until_complete( - sos.send( - Transfer(timestamp=ts, priority=Priority.NOMINAL, transfer_id=12340, fragmented_payload=[]), - loop.time() + 10.0, - ) + sos = SerialOutputSession( + specifier=OutputSessionSpecifier(ServiceDataSpecifier(321, ServiceDataSpecifier.Role.REQUEST), 1111), + payload_metadata=PayloadMetadata(1024), + mtu=10, + local_node_id=None, + send_handler=do_send, + finalizer=do_finalize, ) sos = SerialOutputSession( diff --git a/pyuavcan/transport/serial/_tracer.py b/pyuavcan/transport/serial/_tracer.py index 013aa27af..997a22d8b 100644 --- a/pyuavcan/transport/serial/_tracer.py +++ b/pyuavcan/transport/serial/_tracer.py @@ -55,7 +55,7 @@ def __repr__(self) -> str: fragment = bytes(self.fragment[:limit]).hex() + f"...<+{len(self.fragment) - limit}B>..." else: fragment = bytes(self.fragment).hex() - return pyuavcan.util.repr_attributes(self, str(self.direction).split(".")[-1], fragment) + return pyuavcan.util.repr_attributes(self, self.direction.name, fragment) @dataclasses.dataclass(frozen=True) diff --git a/pyuavcan/transport/udp/_session/_input.py b/pyuavcan/transport/udp/_session/_input.py index 251e9e1c5..3e155f00e 100644 --- a/pyuavcan/transport/udp/_session/_input.py +++ b/pyuavcan/transport/udp/_session/_input.py @@ -110,7 +110,7 @@ def __init__( assert callable(self._maybe_finalizer) self._transfer_id_timeout = self.DEFAULT_TRANSFER_ID_TIMEOUT - self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue(loop=loop) + self._queue: asyncio.Queue[pyuavcan.transport.TransferFrom] = asyncio.Queue() def _process_frame(self, timestamp: Timestamp, source_node_id: int, frame: typing.Optional[UDPFrame]) -> None: """ @@ -143,7 +143,7 @@ async def receive(self, monotonic_deadline: float) -> typing.Optional[pyuavcan.t try: timeout = monotonic_deadline - self._loop.time() if timeout > 0: - transfer = await asyncio.wait_for(self._queue.get(), timeout, loop=self._loop) + transfer = await asyncio.wait_for(self._queue.get(), timeout) else: transfer = self._queue.get_nowait() except (asyncio.TimeoutError, asyncio.QueueEmpty): diff --git a/pyuavcan/transport/udp/_session/_output.py b/pyuavcan/transport/udp/_session/_output.py index eb9dbd314..4092952ac 100644 --- a/pyuavcan/transport/udp/_session/_output.py +++ b/pyuavcan/transport/udp/_session/_output.py @@ -9,7 +9,7 @@ import asyncio import logging import pyuavcan -from pyuavcan.transport import Timestamp +from pyuavcan.transport import Timestamp, ServiceDataSpecifier from .._frame import UDPFrame @@ -82,12 +82,6 @@ def __init__( self._finalizer = finalizer self._feedback_handler: typing.Optional[typing.Callable[[pyuavcan.transport.Feedback], None]] = None self._statistics = pyuavcan.transport.SessionStatistics() - - if not isinstance(self._specifier, pyuavcan.transport.OutputSessionSpecifier) or not isinstance( - self._payload_metadata, pyuavcan.transport.PayloadMetadata - ): # pragma: no cover - raise TypeError("Invalid parameters") - if self._multiplier < 1: # pragma: no cover raise ValueError(f"Invalid transfer multiplier: {self._multiplier}") @@ -97,9 +91,7 @@ def __init__( else True ), "Internal protocol violation: cannot unicast a message transfer" assert ( - specifier.remote_node_id is not None - if isinstance(specifier.data_specifier, pyuavcan.transport.ServiceDataSpecifier) - else True + specifier.remote_node_id is not None if isinstance(specifier.data_specifier, ServiceDataSpecifier) else True ), "Internal protocol violation: cannot broadcast a service transfer" async def send(self, transfer: pyuavcan.transport.Transfer, monotonic_deadline: float) -> bool: @@ -194,7 +186,6 @@ async def _emit( await asyncio.wait_for( self._loop.sock_sendall(self._sock, b"".join((header, payload))), timeout=monotonic_deadline - self._loop.time(), - loop=self._loop, ) # TODO: use socket timestamping when running on Linux (Windows does not support timestamping). diff --git a/requirements.txt b/requirements.txt index 01614455c..2eede9719 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,6 @@ # Include runtime deps. . -.[cli] .[transport_can_pythoncan] .[transport_serial] .[transport_udp] diff --git a/setup.cfg b/setup.cfg index b864da348..e214e2948 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,9 +31,8 @@ classifiers = License :: OSI Approved :: MIT License Programming Language :: Python Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 - Programming Language :: Python :: 3.9 + Operating System :: OS Independent + Typing :: Typed # ======================================== OPTIONAL DEPENDENCIES ======================================== @@ -52,13 +51,6 @@ transport_serial = transport_udp = libpcap >= 1.10.0b15, < 2.0.0 -# Command-line tool. This is not a transport. -# Per ruamel.yaml docs: "For production systems you should pin the version being used with ``ruamel.yaml<=0.15``" -cli = - ruamel.yaml < 0.16 - requests ~= 2.24 - simplejson ~= 3.17 - # ======================================== PACKAGE CONFIGURATION ======================================== [options] @@ -77,12 +69,6 @@ include = pyuavcan pyuavcan.* -[options.entry_points] -# Scripts and their abridged aliases. -console_scripts = - pyuavcan = pyuavcan._cli:main - uvc = pyuavcan._cli:main - [options.package_data] # jingle bells jingle bells # jingle all the way @@ -100,15 +86,9 @@ console_scripts = [tool:pytest] # https://docs.pytest.org/en/latest/pythonpath.html#invoking-pytest-versus-python-m-pytest # - Application is excluded because it requires the uavcan DSDL package to be generated. Hence there are no unit tests. -# - CLI is excluded because it requires additional third-party dependencies. They require separate environment config. -# - Public regulated data types do not contain any testable Python files, it's just a DSDL repo. -# - Demo application scripts cannot be imported; they are designed to be runnable. # - The import_error package is designed to fail to import, so it has to be excluded obviously. norecursedirs = pyuavcan/application - pyuavcan/_cli - tests/public_regulated_data_types - tests/demo tests/util/import_error testpaths = pyuavcan tests python_files = *.py @@ -138,6 +118,8 @@ no_implicit_optional = True warn_redundant_casts = True warn_unused_ignores = True show_error_context = True +strict_equality = True +implicit_reexport = False mypy_path = .test_dsdl_generated diff --git a/sitecustomize.py b/sitecustomize.py deleted file mode 100644 index 984b4a331..000000000 --- a/sitecustomize.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import os -import sys -import pathlib - -OWN_PATH = pathlib.Path(__file__).absolute() - - -def detect_debugger() -> bool: - if sys.gettrace() is not None: - return True - if (os.path.sep + "pydev") in sys.argv[0]: - return True - return False - - -def setup_coverage() -> None: - try: - import coverage # The module may be missing during early stage setup, no need to abort everything. - except ImportError as ex: - print("COVERAGE NOT CONFIGURED:", ex, file=sys.stderr) - else: - # Coverage configuration; see https://coverage.readthedocs.io/en/coverage-4.2/subprocess.html - # This is kind of a big gun because it makes us track coverage of everything we run, even doc generation, - # but it's acceptable. - os.environ["COVERAGE_PROCESS_START"] = str(OWN_PATH.parent / "setup.cfg") - coverage.process_startup() - - -if detect_debugger(): - print("Debugger detected, coverage will not be tracked to avoid interference.") -else: - print(f"Tracking coverage of {sys.argv[0]} with {OWN_PATH}", file=sys.stderr) - setup_coverage() diff --git a/test.ps1 b/test.ps1 index cece77121..d9586593b 100644 --- a/test.ps1 +++ b/test.ps1 @@ -30,7 +30,7 @@ $ncat_proc = Start-Process '.test_deps/ncat.exe' -Args '-vv --broker --listen lo # TESTING # -# The DSDL gen directory shall exist before coverage is invoked, otherwise its coverage won't be tracked. +# The DSDL gen directory shall exist before coverage is invoked if we want to track its coverage. Remove-Item -Recurse -Force ".test_dsdl_generated" -ErrorAction SilentlyContinue New-Item -Path . -Name ".test_dsdl_generated" -ItemType Directory @@ -42,7 +42,7 @@ $test_ok = False For ($i=1; ($i -le $test_attempts) -and -not $test_ok; $i++) { Write-Host "Running the tests, attempt $i of $test_attempts..." - python -m pytest + python -m coverage run -m pytest $test_ok = $? Write-Host "Attempt $i of $test_attempts completed; success: $test_ok" } diff --git a/test.sh b/test.sh index 454b3fa42..3a666b1c5 100755 --- a/test.sh +++ b/test.sh @@ -42,15 +42,6 @@ python -c "import sys; exit('linux' not in sys.platform)" || die "This script ca cd "${0%/*}" || die "Couldn't cd into this script's directory" -# Extend PYTHONPATH to make sitecustomize.py/usercustomize.py importable. -if [[ -z "${PYTHONPATH:-}" ]] -then - export PYTHONPATH="$PWD" -else - export PYTHONPATH="$PYTHONPATH:$PWD" -fi -echo "PYTHONPATH: $PYTHONPATH" - export PYTHONASYNCIODEBUG=1 command -v ncat || die "Please install ncat. On Debian-based: apt install ncat" @@ -85,19 +76,17 @@ sudo setcap cap_net_raw+eip "$(readlink -f $(command -v python))" || die "Could banner TEST EXECUTION -# TODO: run the tests with the minimal dependency configuration. Set up a new environment here. -# Note that we do not invoke coverage.py explicitly here; this is handled by usercustomize.py. Relevant docs: +# Relevant docs: # - https://coverage.readthedocs.io/en/coverage-4.2/subprocess.html # - https://docs.python.org/3/library/site.html log_format='%(asctime)s %(process)5d %(levelname)-8s %(name)s: %(message)s' -pytest --log-format="$log_format" --log-file='main.log' || die "Core PyTest returned $?" -pytest --log-format="$log_format" --log-file='cli.log' pyuavcan/_cli || die "CLI PyTest returned $?" +coverage run -m pytest --log-format="$log_format" || die "PyTest returned $?" # Every time we launch a Python process, a new coverage file is created, so there may be a lot of those, # possibly nested in sub-directories. find ./*/ -name '.coverage*' -type f -print -exec mv {} . \; || die "Could not lift coverage files" ls -l .coverage* -coverage combine || die "Could not combine coverage data" +coverage combine || die "Could not combine coverage data" # Shall it be desired to measure coverage of the generated code, it is necessary to ensure that the target # directory where the generated code is stored exists before the coverage utility is invoked. @@ -113,8 +102,7 @@ banner STATIC ANALYSIS rm -rf .mypy_cache/ &> /dev/null echo 'YOU SHALL NOT PASS' > .mypy_cache chmod 444 .mypy_cache -mypy --strict --strict-equality --no-implicit-reexport --config-file=setup.cfg pyuavcan tests .test_dsdl_generated \ - || die "MyPy returned $?" +mypy --strict pyuavcan tests .test_dsdl_generated demo || die "MyPy returned $?" # See configuration file for details. black --check . || die "black returned $?" diff --git a/tests/__init__.py b/tests/__init__.py index a9a31f9c3..1780c16cd 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -5,4 +5,6 @@ import os as _os from . import dsdl as dsdl -assert _os.environ.get("PYTHONASYNCIODEBUG", False), "PYTHONASYNCIODEBUG should be set while running the tests" +assert ("PYTHONASYNCIODEBUG" in _os.environ) or ( + _os.environ.get("IGNORE_PYTHONASYNCIODEBUG", False) +), "PYTHONASYNCIODEBUG should be set while running the tests" diff --git a/tests/application/node.py b/tests/application/node.py new file mode 100644 index 000000000..275d61919 --- /dev/null +++ b/tests/application/node.py @@ -0,0 +1,93 @@ +# Copyright (c) 2020 UAVCAN Consortium +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +import typing +import asyncio +import pytest +import pyuavcan +from pyuavcan.transport.udp import UDPTransport +from pyuavcan.transport.redundant import RedundantTransport +from pyuavcan.presentation import Presentation + + +# noinspection PyProtectedMember +@pytest.mark.asyncio # type: ignore +async def _unittest_slow_node(generated_packages: typing.List[pyuavcan.dsdl.GeneratedPackageInfo]) -> None: + from pyuavcan.application import Node + from uavcan.node import Version_1_0, Heartbeat_1_0, GetInfo_1_0, Mode_1_0, Health_1_0 + + asyncio.get_running_loop().slow_callback_duration = 3.0 + + assert generated_packages + remote_pres = Presentation(UDPTransport("127.1.1.1")) + remote_hb_sub = remote_pres.make_subscriber_with_fixed_subject_id(Heartbeat_1_0) + remote_info_cln = remote_pres.make_client_with_fixed_service_id(GetInfo_1_0, 258) + + trans = RedundantTransport() + pres = Presentation(trans) + try: + info = GetInfo_1_0.Response( + protocol_version=Version_1_0(*pyuavcan.UAVCAN_SPECIFICATION_VERSION), + software_version=Version_1_0(*pyuavcan.__version_info__[:2]), + name="org.uavcan.pyuavcan.test.node", + ) + node = Node(pres, info, with_diagnostic_subscriber=True) + print("node:", node) + assert node.presentation is pres + node.start() + node.start() # Idempotency + + node.heartbeat_publisher.priority = pyuavcan.transport.Priority.FAST + node.heartbeat_publisher.period = 0.5 + node.heartbeat_publisher.mode = Mode_1_0.MAINTENANCE # type: ignore + node.heartbeat_publisher.health = Health_1_0.ADVISORY # type: ignore + node.heartbeat_publisher.vendor_specific_status_code = 93 + with pytest.raises(ValueError): + node.heartbeat_publisher.period = 99.0 + with pytest.raises(ValueError): + node.heartbeat_publisher.vendor_specific_status_code = -299 + + assert node.heartbeat_publisher.priority == pyuavcan.transport.Priority.FAST + assert node.heartbeat_publisher.period == pytest.approx(0.5) + assert node.heartbeat_publisher.mode == Mode_1_0.MAINTENANCE + assert node.heartbeat_publisher.health == Health_1_0.ADVISORY + assert node.heartbeat_publisher.vendor_specific_status_code == 93 + + assert None is await remote_hb_sub.receive_for(2.0) + + assert trans.local_node_id is None + trans.attach_inferior(UDPTransport("127.1.1.2")) + assert trans.local_node_id == 258 + + for _ in range(2): + hb_transfer = await remote_hb_sub.receive_for(2.0) + assert hb_transfer is not None + hb, transfer = hb_transfer + assert transfer.source_node_id == 258 + assert transfer.priority == pyuavcan.transport.Priority.FAST + assert 1 <= hb.uptime <= 9 + assert hb.mode.value == Mode_1_0.MAINTENANCE + assert hb.health.value == Health_1_0.ADVISORY + assert hb.vendor_specific_status_code == 93 + + info_transfer = await remote_info_cln.call(GetInfo_1_0.Request()) + assert info_transfer is not None + resp, transfer = info_transfer + assert transfer.source_node_id == 258 + assert isinstance(resp, GetInfo_1_0.Response) + assert resp.name.tobytes().decode() == "org.uavcan.pyuavcan.test.node" + assert resp.protocol_version.major == pyuavcan.UAVCAN_SPECIFICATION_VERSION[0] + assert resp.software_version.major == pyuavcan.__version_info__[0] + + trans.detach_inferior(trans.inferiors[0]) + assert trans.local_node_id is None + + assert None is await remote_hb_sub.receive_for(2.0) + + node.close() + node.close() # Idempotency + finally: + pres.close() + remote_pres.close() + await asyncio.sleep(1.0) # Let the background tasks terminate. diff --git a/tests/cli/README.md b/tests/cli/README.md deleted file mode 100644 index 5c896fc62..000000000 --- a/tests/cli/README.md +++ /dev/null @@ -1,5 +0,0 @@ -CLI tests -========= - -The CLI tests are executed against the demo scripts in order to test two components at once: -the CLI tool and the demo scripts. diff --git a/tests/cli/__init__.py b/tests/cli/__init__.py deleted file mode 100644 index 77d747e42..000000000 --- a/tests/cli/__init__.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import sys -import typing -import dataclasses - - -@dataclasses.dataclass(frozen=True) -class TransportConfig: - cli_args: typing.Sequence[str] - can_transmit: bool - - -TransportFactory = typing.Callable[[typing.Optional[int]], TransportConfig] -""" -This factory constructs arguments for the CLI instructing it to use a particular transport configuration. -The factory takes one argument - the node-ID - which can be None (anonymous). -""" - - -def _make_transport_factories_for_cli() -> typing.Iterable[TransportFactory]: - """ - Sensible transport configurations supported by the CLI to test against. - Don't forget to extend when adding support for new transports. - """ - if sys.platform == "linux": - # CAN via SocketCAN - yield lambda nid: TransportConfig( - cli_args=(f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan0",64),local_node_id={nid})',), - can_transmit=True, - ) - - # Redundant CAN via SocketCAN - yield lambda nid: TransportConfig( - cli_args=( - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan0",8),local_node_id={nid})', - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan1",32),local_node_id={nid})', - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan2",64),local_node_id={nid})', - ), - can_transmit=True, - ) - - # Serial via TCP/IP tunnel (emulation) - from tests.transport.serial import VIRTUAL_BUS_URI - - yield lambda nid: TransportConfig( - cli_args=(f'--tr=Serial("{VIRTUAL_BUS_URI}",local_node_id={nid})',), - can_transmit=True, - ) - - # UDP/IP on localhost (cannot transmit if anonymous) - yield lambda nid: TransportConfig( - cli_args=(f'--tr=UDP("127.0.0.{nid}")',), - can_transmit=True, - ) if nid is not None else TransportConfig( - cli_args=('--tr=UDP("127.0.0.1",anonymous=True)',), - can_transmit=False, - ) - - # Redundant UDP+Serial. The UDP transport does not support anonymous transfers. - yield lambda nid: TransportConfig( - cli_args=( - f'--tr=Serial("{VIRTUAL_BUS_URI}",local_node_id={nid})', - (f'--tr=UDP("127.0.0.{nid}")' if nid is not None else '--tr=UDP("127.0.0.1",anonymous=True)'), - ), - can_transmit=nid is not None, - ) - - -TRANSPORT_FACTORIES = list(_make_transport_factories_for_cli()) diff --git a/tests/cli/_call.py b/tests/cli/_call.py deleted file mode 100644 index 4799b5086..000000000 --- a/tests/cli/_call.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import json -import pytest -import pathlib -import subprocess -import pyuavcan -from tests.dsdl.conftest import PUBLIC_REGULATED_DATA_TYPES_DIR -from ._subprocess import run_cli_tool - - -def _unittest_slow_cli_call_a() -> None: - # Generate DSDL namespace "uavcan" - if not pathlib.Path("uavcan").exists(): - run_cli_tool("dsdl-gen-pkg", str(PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan")) - - result_text = run_cli_tool( - "-v", "call", "1234", "uavcan.node.GetInfo.1.0", "{}", "--tr=Loopback(1234)", "--format", "json" - ) - result = json.loads(result_text) - assert result["430"]["name"] == "org.uavcan.pyuavcan.cli.call" - assert result["430"]["protocol_version"]["major"] == pyuavcan.UAVCAN_SPECIFICATION_VERSION[0] - assert result["430"]["protocol_version"]["minor"] == pyuavcan.UAVCAN_SPECIFICATION_VERSION[1] - assert result["430"]["software_version"]["major"] == pyuavcan.__version_info__[0] - assert result["430"]["software_version"]["minor"] == pyuavcan.__version_info__[1] - - with pytest.raises(subprocess.CalledProcessError): - # Will time out because we're using a wrong service-ID - run_cli_tool("-v", "call", "1234", "123.uavcan.node.GetInfo.1.0", "{}", "--tr=Loopback(1234)") diff --git a/tests/cli/_demo_app.py b/tests/cli/_demo_app.py deleted file mode 100644 index e445de917..000000000 --- a/tests/cli/_demo_app.py +++ /dev/null @@ -1,322 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import re -import sys -import time -import json -import typing -import pytest -import pathlib -import dataclasses -import pyuavcan -from ._subprocess import run_cli_tool, BackgroundChildProcess, DEMO_DIR - -# noinspection PyProtectedMember -from pyuavcan._cli import DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL -from tests.dsdl.conftest import TEST_DATA_TYPES_DIR, PUBLIC_REGULATED_DATA_TYPES_DIR, generated_packages - - -@dataclasses.dataclass -class _IfaceOption: - demo_env_vars: typing.Dict[str, str] - make_cli_args: typing.Callable[[typing.Optional[int]], typing.Sequence[str]] - - -def _get_iface_options() -> typing.Iterable[_IfaceOption]: - """ - Provides interface options to test the demo against. - When adding new transports, add them to the demo and update this factory accordingly. - Don't forget about redundant configurations, too. - """ - if sys.platform == "linux": - # CAN - yield _IfaceOption( - demo_env_vars={"DEMO_INTERFACE_KIND": "can"}, - make_cli_args=lambda nid: ( # The demo uses Classic CAN! SocketCAN does not support nonuniform MTU well. - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan0",8),local_node_id={nid})', - ), - ) - - # TMR CAN - yield _IfaceOption( - demo_env_vars={"DEMO_INTERFACE_KIND": "can_can_can"}, - make_cli_args=lambda nid: ( # The MTU values are like in the demo otherwise SocketCAN may misbehave. - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan0",8),local_node_id={nid})', - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan1",32),local_node_id={nid})', - f'--tr=CAN(can.media.socketcan.SocketCANMedia("vcan2",64),local_node_id={nid})', - ), - ) - - # Serial - yield _IfaceOption( - demo_env_vars={"DEMO_INTERFACE_KIND": "serial"}, - make_cli_args=lambda nid: (f'--tr=Serial("socket://localhost:50905",local_node_id={nid})',), - ) - - # UDP - yield _IfaceOption( - demo_env_vars={"DEMO_INTERFACE_KIND": "udp"}, - make_cli_args=lambda nid: ( - (f'--tr=UDP("127.0.0.{nid}")',) # Regular node - if nid is not None - else (f'--tr=UDP("127.0.0.1",anonymous=True)',) # Anonymous node - ), - ) - - # DMR UDP+Serial - yield _IfaceOption( - demo_env_vars={"DEMO_INTERFACE_KIND": "udp_serial"}, - make_cli_args=lambda nid: ( - ( - f'--tr=UDP("127.0.0.{nid}")' # Regular node - if nid is not None - else f'--tr=UDP("127.0.0.1",anonymous=True)' # Anonymous node - ), - f'--tr=Serial("socket://localhost:50905",local_node_id={nid})', - ), - ) - - -@pytest.mark.parametrize("iface_option", _get_iface_options()) # type: ignore -def _unittest_slow_cli_demo_app( - generated_packages: typing.Iterator[typing.List[pyuavcan.dsdl.GeneratedPackageInfo]], iface_option: _IfaceOption -) -> None: - """ - This test is KINDA FRAGILE. It makes assumptions about particular data types and their port IDs and other - aspects of the demo application. If you change things in the demo, this test will likely break. - """ - import uavcan.node - - del generated_packages - try: - pathlib.Path("/tmp/dsdl-for-my-program").rmdir() # Where the demo script puts its generated packages - except OSError: - pass - - # The demo may need to generate packages as well, so we launch it first. - demo_proc_env_vars = iface_option.demo_env_vars.copy() - demo_proc_env_vars["PYUAVCAN_LOGLEVEL"] = "DEBUG" - demo_proc = BackgroundChildProcess( - "python", str(DEMO_DIR / "demo_app.py"), environment_variables=demo_proc_env_vars - ) - assert demo_proc.alive - - # Generate DSDL namespace "sirius_cyber_corp" - if not pathlib.Path("sirius_cyber_corp").exists(): - run_cli_tool( - "dsdl-gen-pkg", - str(TEST_DATA_TYPES_DIR / "sirius_cyber_corp"), - "--lookup", - DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL, - ) - - # Generate DSDL namespace "test" - if not pathlib.Path("test_dsdl_namespace").exists(): - run_cli_tool( - "dsdl-gen-pkg", - str(TEST_DATA_TYPES_DIR / "test_dsdl_namespace"), - "--lookup", - DEFAULT_PUBLIC_REGULATED_DATA_TYPES_ARCHIVE_URL, - ) - - # Generate DSDL namespace "uavcan" - if not pathlib.Path("uavcan").exists(): - run_cli_tool("dsdl-gen-pkg", str(PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan")) - - # Time to let the background processes finish initialization. - # The usage demo might take a long time to start because it may have to generate packages first. - time.sleep(90) - - proc_sub_heartbeat = BackgroundChildProcess.cli( - "sub", - "uavcan.node.Heartbeat.1.0", - "--format=json", # Count unlimited - "--with-metadata", - *iface_option.make_cli_args(None), # type: ignore - ) - - proc_sub_temperature = BackgroundChildProcess.cli( - "sub", - "2345.uavcan.si.sample.temperature.Scalar.1.0", - "--count=3", - "--format=json", - "--with-metadata", - *iface_option.make_cli_args(None), # type: ignore - ) - - proc_sub_diagnostic = BackgroundChildProcess.cli( - "sub", - "uavcan.diagnostic.Record.1.1", - "--count=3", - "--format=json", - "--with-metadata", - *iface_option.make_cli_args(None), # type: ignore - ) - - try: - assert demo_proc.alive - - run_cli_tool( - "-v", - "pub", - "2345.uavcan.si.sample.temperature.Scalar.1.0", - "{kelvin: 321.5}", - "--count=5", - "--period=1", - "--priority=slow", - "--heartbeat-fields={vendor_specific_status_code: 123}", - *iface_option.make_cli_args(1), # type: ignore - timeout=10.0, - ) - - time.sleep(1.0) # Time to sync up - - out_sub_heartbeat = proc_sub_heartbeat.wait(2.0, interrupt=True)[1].splitlines() - out_sub_temperature = proc_sub_temperature.wait(2.0, interrupt=True)[1].splitlines() - out_sub_diagnostic = proc_sub_diagnostic.wait(2.0, interrupt=True)[1].splitlines() - - assert demo_proc.alive - # Run service tests while the demo process is still running. - node_info_text = run_cli_tool( - "-v", - "call", - "42", - "uavcan.node.GetInfo.1.0", - "{}", - "--format", - "json", - "--with-metadata", - "--priority", - "slow", - "--timeout", - "3.0", - *iface_option.make_cli_args(123), # type: ignore - timeout=5.0, - ) - assert demo_proc.alive - print("node_info_text:", node_info_text) - node_info = json.loads(node_info_text) - assert node_info["430"]["_metadata_"]["source_node_id"] == 42 - assert node_info["430"]["_metadata_"]["transfer_id"] >= 0 - assert "slow" in node_info["430"]["_metadata_"]["priority"].lower() - assert node_info["430"]["name"] == "org.uavcan.pyuavcan.demo.demo_app" - assert node_info["430"]["protocol_version"]["major"] == pyuavcan.UAVCAN_SPECIFICATION_VERSION[0] - assert node_info["430"]["protocol_version"]["minor"] == pyuavcan.UAVCAN_SPECIFICATION_VERSION[1] - - assert demo_proc.alive - command_response = json.loads( - run_cli_tool( - "-v", - "call", - "42", - "uavcan.node.ExecuteCommand.1.1", - f"{{command: {uavcan.node.ExecuteCommand_1_1.Request.COMMAND_STORE_PERSISTENT_STATES} }}", - "--format", - "json", - *iface_option.make_cli_args(123), # type: ignore - timeout=5.0, - ) - ) - assert command_response["435"]["status"] == uavcan.node.ExecuteCommand_1_1.Response.STATUS_BAD_COMMAND - - # Next request - this fails if the OUTPUT TRANSFER-ID MAP save/restore logic is not working. - command_response = json.loads( - run_cli_tool( - "-v", - "call", - "42", - "uavcan.node.ExecuteCommand.1.1", - "{command: 23456}", - "--format", - "json", - *iface_option.make_cli_args(123), # type: ignore - timeout=5.0, - ) - ) - assert command_response["435"]["status"] == uavcan.node.ExecuteCommand_1_1.Response.STATUS_SUCCESS - - assert demo_proc.alive - least_squares_response = json.loads( - run_cli_tool( - "-vv", - "call", - "42", - "123.sirius_cyber_corp.PerformLinearLeastSquaresFit.1.0", - "{points: [{x: 1, y: 2}, {x: 10, y: 20}]}", - "--timeout=5", - "--format", - "json", - *iface_option.make_cli_args(123), # type: ignore - timeout=6.0, - ) - ) - assert least_squares_response["123"]["slope"] == pytest.approx(2.0) - assert least_squares_response["123"]["y_intercept"] == pytest.approx(0.0) - - assert demo_proc.alive - # Next request - this fails if the OUTPUT TRANSFER-ID MAP save/restore logic is not working. - command_response = json.loads( - run_cli_tool( - "-v", - "call", - "42", - "uavcan.node.ExecuteCommand.1.1", - f"{{command: {uavcan.node.ExecuteCommand_1_1.Request.COMMAND_POWER_OFF} }}", - "--format", - "json", - *iface_option.make_cli_args(123), # type: ignore - timeout=5.0, - ) - ) - assert command_response["435"]["status"] == uavcan.node.ExecuteCommand_1_1.Response.STATUS_SUCCESS - - # We've just asked the node to terminate, wait for it here. - out_demo_proc = demo_proc.wait(10.0)[1].splitlines() - - print("out_demo_proc:", *out_demo_proc, sep="\n\t") - print("out_sub_heartbeat:", *out_sub_heartbeat, sep="\n\t") - print("out_sub_temperature:", *out_sub_temperature, sep="\n\t") - print("out_sub_diagnostic:", *out_sub_diagnostic, sep="\n\t") - - assert out_demo_proc - assert any(re.match(r"TEMPERATURE \d+\.\d+ C", s) for s in out_demo_proc) - - # We receive three heartbeats in order to eliminate possible edge cases due to timing jitter. - # Sort by source node ID and eliminate the middle; thus we eliminate the uncertainty. - heartbeats_ordered_by_nid = list( - sorted( - (json.loads(s) for s in out_sub_heartbeat), key=lambda x: int(x["7509"]["_metadata_"]["source_node_id"]) - ) - ) - print("heartbeats_ordered_by_nid:", heartbeats_ordered_by_nid) - heartbeat_pub, heartbeat_demo = heartbeats_ordered_by_nid[0], heartbeats_ordered_by_nid[-1] - print("heartbeat_pub :", heartbeat_pub) - print("heartbeat_demo:", heartbeat_demo) - - assert "slow" in heartbeat_pub["7509"]["_metadata_"]["priority"].lower() - assert heartbeat_pub["7509"]["_metadata_"]["transfer_id"] >= 0 - assert heartbeat_pub["7509"]["_metadata_"]["source_node_id"] == 1 - assert heartbeat_pub["7509"]["uptime"] in (0, 1) - assert heartbeat_pub["7509"]["vendor_specific_status_code"] == 123 - - assert "nominal" in heartbeat_demo["7509"]["_metadata_"]["priority"].lower() - assert heartbeat_demo["7509"]["_metadata_"]["source_node_id"] == 42 - assert heartbeat_demo["7509"]["vendor_specific_status_code"] == demo_proc.pid % 100 - - for parsed in (json.loads(s) for s in out_sub_temperature): - assert "slow" in parsed["2345"]["_metadata_"]["priority"].lower() - assert parsed["2345"]["_metadata_"]["transfer_id"] >= 0 - assert parsed["2345"]["_metadata_"]["source_node_id"] == 1 - assert parsed["2345"]["kelvin"] == pytest.approx(321.5) - - assert len(out_sub_diagnostic) >= 1 - finally: - # It is important to get rid of processes even in the event of failure because if we fail to do so - # the processes running in the background may fail the following tests, possibly making them very hard - # to diagnose and debug. - demo_proc.kill() - proc_sub_heartbeat.kill() - proc_sub_temperature.kill() - proc_sub_diagnostic.kill() diff --git a/tests/cli/_pick_node_id.py b/tests/cli/_pick_node_id.py deleted file mode 100644 index 7b22179c5..000000000 --- a/tests/cli/_pick_node_id.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import pytest -from ._subprocess import run_cli_tool, BackgroundChildProcess, CalledProcessError -from . import TRANSPORT_FACTORIES, TransportFactory - - -@pytest.mark.parametrize("transport_factory", TRANSPORT_FACTORIES) # type: ignore -def _unittest_slow_cli_pick_nid(transport_factory: TransportFactory) -> None: - # We spawn a lot of processes here, which might strain the test system a little, so beware. I've tested it - # with 120 processes and it made my workstation (24 GB RAM ~4 GHz Core i7) struggle to the point of being - # unable to maintain sufficiently real-time operation for the test to pass. Hm. - used_node_ids = list(range(10)) - pubs = [ - BackgroundChildProcess.cli( - "pub", - "--period=0.4", - "--count=200", - # Construct an environment variable to ensure syntax equivalency with the `--transport=...` CLI args. - environment_variables={ - "PYUAVCAN_CLI_TRANSPORT": (",".join(x.replace("--tr=", "") for x in transport_factory(idx).cli_args)) - }, - ) - for idx in used_node_ids - ] - result = run_cli_tool("-v", "pick-nid", *transport_factory(None).cli_args, timeout=100.0) - print("pick-nid result:", result) - assert int(result) not in used_node_ids - for p in pubs: - p.wait(100.0, interrupt=True) - - -def _unittest_slow_cli_pick_nid_loopback() -> None: - result = run_cli_tool( - "-v", - "pick-nid", - timeout=30.0, - environment_variables={"PYUAVCAN_CLI_TRANSPORT": "[Loopback(None), Loopback(None)]"}, - ) - print("pick-nid result:", result) - assert 0 <= int(result) < 2 ** 64 - - -def _unittest_slow_cli_pick_nid_udp_localhost() -> None: - result = run_cli_tool( - "-v", - "pick-nid", - timeout=30.0, - environment_variables={"PYUAVCAN_CLI_TRANSPORT": 'UDP("127.0.0.1",anonymous=True)'}, - ) - print("pick-nid result:", result) - # Exclude zero from the set because an IP address with the host address of zero may cause complications. - assert 1 <= int(result) <= 65534 - - with pytest.raises(CalledProcessError): - # Fails because the transport is not anonymous! - run_cli_tool("-v", "pick-nid", '--tr=UDP("127.0.0.123")', timeout=30.0) diff --git a/tests/cli/_pub_sub.py b/tests/cli/_pub_sub.py deleted file mode 100644 index ca7cd04a4..000000000 --- a/tests/cli/_pub_sub.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import time -import json -import pathlib -import pytest -from tests.dsdl.conftest import PUBLIC_REGULATED_DATA_TYPES_DIR -from ._subprocess import run_cli_tool, BackgroundChildProcess -from . import TRANSPORT_FACTORIES, TransportFactory - - -@pytest.mark.parametrize("transport_factory", TRANSPORT_FACTORIES) # type: ignore -def _unittest_slow_cli_pub_sub(transport_factory: TransportFactory) -> None: - # Generate DSDL namespace "uavcan" - if not pathlib.Path("uavcan").exists(): - run_cli_tool("dsdl-gen-pkg", str(PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan")) - - proc_sub_heartbeat = BackgroundChildProcess.cli( - "sub", - "uavcan.node.Heartbeat.1.0", - "--format=json", # Count unlimited - "--with-metadata", - *transport_factory(None).cli_args, - ) - - proc_sub_diagnostic = BackgroundChildProcess.cli( - "sub", - "4321.uavcan.diagnostic.Record.1.1", - "--count=3", - "--format=json", - "--with-metadata", - *transport_factory(None).cli_args, - ) - - proc_sub_diagnostic_wrong_pid = BackgroundChildProcess.cli( - "sub", - "uavcan.diagnostic.Record.1.1", - "--count=3", - "--format=yaml", - "--with-metadata", - *transport_factory(None).cli_args, - ) - - proc_sub_temperature = BackgroundChildProcess.cli( - "sub", - "555.uavcan.si.sample.temperature.Scalar.1.0", - "--count=3", - "--format=json", - *transport_factory(None).cli_args, - ) - - time.sleep(1.0) # Time to let the background processes finish initialization - - run_cli_tool( - "-v", - "pub", - "4321.uavcan.diagnostic.Record.1.1", - '{severity: {value: 6}, timestamp: {microsecond: 123456}, text: "Hello world!"}', - "1234.uavcan.diagnostic.Record.1.1", - '{text: "Goodbye world."}', - "555.uavcan.si.sample.temperature.Scalar.1.0", - "{kelvin: 123.456}", - "--count=3", - "--period=0.1", - "--priority=slow", - "--heartbeat-fields={vendor_specific_status_code: 54}", - *transport_factory(51).cli_args, - timeout=10.0, - ) - - time.sleep(1.0) # Time to sync up - - out_sub_heartbeat = proc_sub_heartbeat.wait(1.0, interrupt=True)[1].splitlines() - out_sub_diagnostic = proc_sub_diagnostic.wait(1.0, interrupt=True)[1].splitlines() - out_sub_temperature = proc_sub_temperature.wait(1.0, interrupt=True)[1].splitlines() - - print("out_sub_heartbeat:", *out_sub_heartbeat, sep="\n\t") - print("out_sub_diagnostic:", *out_sub_diagnostic, sep="\n\t") - print("proc_sub_temperature:", *out_sub_temperature, sep="\n\t") - - heartbeats = list(map(json.loads, out_sub_heartbeat)) - diagnostics = list(map(json.loads, out_sub_diagnostic)) - temperatures = list(map(json.loads, out_sub_temperature)) - - print("heartbeats:", *heartbeats, sep="\n\t") - print("diagnostics:", *diagnostics, sep="\n\t") - print("temperatures:", *temperatures, sep="\n\t") - - assert 2 <= len(heartbeats) <= 6 - for m in heartbeats: - assert "slow" in m["7509"]["_metadata_"]["priority"].lower() - assert m["7509"]["_metadata_"]["transfer_id"] >= 0 - assert m["7509"]["_metadata_"]["source_node_id"] == 51 - assert m["7509"]["uptime"] in (0, 1) - assert m["7509"]["vendor_specific_status_code"] == 54 - - assert len(diagnostics) == 3 - for m in diagnostics: - assert "slow" in m["4321"]["_metadata_"]["priority"].lower() - assert m["4321"]["_metadata_"]["transfer_id"] >= 0 - assert m["4321"]["_metadata_"]["source_node_id"] == 51 - assert m["4321"]["timestamp"]["microsecond"] == 123456 - assert m["4321"]["text"] == "Hello world!" - - assert len(temperatures) == 3 - assert all(map(lambda mt: mt["555"]["kelvin"] == pytest.approx(123.456), temperatures)) - - assert proc_sub_diagnostic_wrong_pid.alive - assert proc_sub_diagnostic_wrong_pid.wait(1.0, interrupt=True)[1].strip() == "" - - -@pytest.mark.parametrize("transport_factory", TRANSPORT_FACTORIES) # type: ignore -def _unittest_slow_cli_pub_sub_anon(transport_factory: TransportFactory) -> None: - # Generate DSDL namespace "uavcan" - if not pathlib.Path("uavcan").exists(): - run_cli_tool("dsdl-gen-pkg", str(PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan")) - - proc_sub_heartbeat = BackgroundChildProcess.cli( - "-v", "sub", "uavcan.node.Heartbeat.1.0", "--format=json", *transport_factory(None).cli_args # Count unlimited - ) - - proc_sub_diagnostic_with_meta = BackgroundChildProcess.cli( - "-v", - "sub", - "uavcan.diagnostic.Record.1.1", - "--format=json", - "--with-metadata", - *transport_factory(None).cli_args, - ) - - proc_sub_diagnostic_no_meta = BackgroundChildProcess.cli( - "-v", - "sub", - "uavcan.diagnostic.Record.1.1", - "--format=json", - *transport_factory(None).cli_args, - ) - - time.sleep(3.0) # Time to let the background processes finish initialization - - if transport_factory(None).can_transmit: - proc = BackgroundChildProcess.cli( - "-v", - "pub", - "uavcan.diagnostic.Record.1.1", - "{}", - "--count=2", - "--period=2", - *transport_factory(None).cli_args, - ) - proc.wait(timeout=8) - - time.sleep(2.0) # Time to sync up - - assert ( - proc_sub_heartbeat.wait(1.0, interrupt=True)[1].strip() == "" - ), "Anonymous nodes must not broadcast heartbeat" - - diagnostics = list( - json.loads(s) for s in proc_sub_diagnostic_with_meta.wait(1.0, interrupt=True)[1].splitlines() - ) - print("diagnostics:", diagnostics) - # Remember that anonymous transfers over redundant transports are NOT deduplicated. - # Hence, to support the case of redundant transports, we use 'greater or equal' here. - assert len(diagnostics) >= 2 - for m in diagnostics: - assert "nominal" in m["8184"]["_metadata_"]["priority"].lower() - assert m["8184"]["_metadata_"]["transfer_id"] >= 0 - assert m["8184"]["_metadata_"]["source_node_id"] is None - assert m["8184"]["timestamp"]["microsecond"] == 0 - assert m["8184"]["text"] == "" - - diagnostics = list(json.loads(s) for s in proc_sub_diagnostic_no_meta.wait(1.0, interrupt=True)[1].splitlines()) - print("diagnostics:", diagnostics) - assert len(diagnostics) >= 2 # >= because see above - for index, m in enumerate(diagnostics): - assert m["8184"]["timestamp"]["microsecond"] == 0 - assert m["8184"]["text"] == "" - else: - proc = BackgroundChildProcess.cli( - "-v", - "pub", - "uavcan.diagnostic.Record.1.1", - "{}", - "--count=2", - "--period=2", - *transport_factory(None).cli_args, - ) - assert 0 < proc.wait(timeout=8)[0] diff --git a/tests/cli/_trivial.py b/tests/cli/_trivial.py deleted file mode 100644 index 2a335a94d..000000000 --- a/tests/cli/_trivial.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2019 UAVCAN Consortium -# This software is distributed under the terms of the MIT License. -# Author: Pavel Kirienko - -import pytest -import subprocess -from ._subprocess import run_cli_tool - - -_COMMANDS = ["dsdl-generate-packages", "call", "pick-node-id", "publish", "show-transport", "subscribe"] - - -def _unittest_cli_help() -> None: - # Just make sure that the help can be displayed without issues. - # I once encountered a super obscure failure where I added a line like "(PID % 100)" into a help string - # and the option --help starting failing in the most obscure way possible because the part "% 100)" was - # interpreted as a format specifier. The Python's built-in argparse library is unsuitable for complex - # applications, debugging it is a pain. - # Anyway, so here we just make sure that we can print help for every CLI command. - run_cli_tool("--help", timeout=10.0) - for cmd in _COMMANDS: - run_cli_tool(cmd, "--help", timeout=10.0) - - -def _unittest_trivial() -> None: - run_cli_tool("show-transport", timeout=2.0) - - with pytest.raises(subprocess.CalledProcessError): - run_cli_tool(timeout=2.0) - - with pytest.raises(subprocess.CalledProcessError): - run_cli_tool("invalid-command", timeout=2.0) - - with pytest.raises(subprocess.CalledProcessError): - run_cli_tool("dsdl-gen-pkg", "nonexistent/path", timeout=2.0) - - with pytest.raises(subprocess.CalledProcessError): # Look-up of a nonexistent package requires large timeout - run_cli_tool("pub", "nonexistent.data.Type.1.0", "{}", "--tr=Loopback(None)", timeout=5.0) diff --git a/tests/demo/README.md b/tests/demo/README.md deleted file mode 100644 index 008b233c5..000000000 --- a/tests/demo/README.md +++ /dev/null @@ -1,5 +0,0 @@ -PyUAVCAN usage demos -==================== - -This directory contains demo programs. -They are kept with the tests because they are also continuously tested and validated. diff --git a/tests/demo/__init__.py b/tests/demo/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/demo/_demo_app.py b/tests/demo/_demo_app.py new file mode 100644 index 000000000..9f3db2e6a --- /dev/null +++ b/tests/demo/_demo_app.py @@ -0,0 +1,317 @@ +# Copyright (c) 2020 UAVCAN Consortium +# This software is distributed under the terms of the MIT License. +# Author: Pavel Kirienko + +import os +import re +import sys +import shutil +import typing +import pytest +import asyncio +import pathlib +import tempfile +import dataclasses +import pyuavcan +from ._subprocess import BackgroundChildProcess + +# noinspection PyUnresolvedReferences +from tests.dsdl.conftest import generated_packages + + +DEMO_APP_NODE_ID = 42 +DEMO_DIR = pathlib.Path(__file__).absolute().parent.parent.parent / "demo" + + +@dataclasses.dataclass +class RunConfig: + demo_env_vars: typing.Dict[str, str] + local_transport_factory: typing.Callable[[typing.Optional[int]], pyuavcan.transport.Transport] + + +def _get_run_configs() -> typing.Iterable[RunConfig]: + """ + Provides interface options to test the demo against. + When adding new transports, add them to the demo and update this factory accordingly. + Don't forget about redundant configurations, too. + """ + from pyuavcan.transport.redundant import RedundantTransport + from pyuavcan.transport.serial import SerialTransport + from pyuavcan.transport.udp import UDPTransport + + # UDP + yield RunConfig( + demo_env_vars={"DEMO_INTERFACE_KIND": "udp"}, + local_transport_factory=lambda nid: UDPTransport(f"127.0.0.{1 if nid is None else nid}", anonymous=nid is None), + ) + + # Serial + yield RunConfig( + demo_env_vars={"DEMO_INTERFACE_KIND": "serial"}, + local_transport_factory=lambda nid: SerialTransport("socket://localhost:50905", local_node_id=nid), + ) + + # DMR UDP+Serial + def make_udp_serial(nid: typing.Optional[int]) -> pyuavcan.transport.Transport: + tr = RedundantTransport() + if nid is not None: + tr.attach_inferior(UDPTransport(f"127.0.0.{nid}")) + else: + tr.attach_inferior(UDPTransport(f"127.0.0.1", anonymous=True)) + tr.attach_inferior(SerialTransport("socket://localhost:50905", local_node_id=nid)) + return tr + + yield RunConfig( + demo_env_vars={"DEMO_INTERFACE_KIND": "udp_serial"}, + local_transport_factory=make_udp_serial, + ) + + if sys.platform.startswith("linux"): + from pyuavcan.transport.can.media.socketcan import SocketCANMedia + from pyuavcan.transport.can import CANTransport + + # CAN + yield RunConfig( + demo_env_vars={"DEMO_INTERFACE_KIND": "can"}, + # The demo uses Classic CAN! SocketCAN does not support nonuniform MTU well. + local_transport_factory=lambda nid: CANTransport(SocketCANMedia("vcan0", 8), local_node_id=nid), + ) + + # TMR CAN + def make_tmr_can(nid: typing.Optional[int]) -> pyuavcan.transport.Transport: + from pyuavcan.transport.redundant import RedundantTransport + + tr = RedundantTransport() + tr.attach_inferior(CANTransport(SocketCANMedia("vcan0", 8), local_node_id=nid)) + tr.attach_inferior(CANTransport(SocketCANMedia("vcan1", 32), local_node_id=nid)) + tr.attach_inferior(CANTransport(SocketCANMedia("vcan2", 64), local_node_id=nid)) + return tr + + yield RunConfig( + demo_env_vars={"DEMO_INTERFACE_KIND": "can_can_can"}, + local_transport_factory=make_tmr_can, + ) + + +@pytest.mark.parametrize("parameters", [(idx == 0, rc) for idx, rc in enumerate(_get_run_configs())]) # type: ignore +@pytest.mark.asyncio # type: ignore +async def _unittest_slow_demo_app( + generated_packages: typing.Iterator[typing.List[pyuavcan.dsdl.GeneratedPackageInfo]], + parameters: typing.Tuple[bool, RunConfig], +) -> None: + """ + This test is KINDA FRAGILE. It makes assumptions about particular data types and their port IDs and other + aspects of the demo application. If you change things in the demo, this test will likely break. + """ + import uavcan.node + import uavcan.diagnostic + import uavcan.si.sample.temperature + import sirius_cyber_corp + import pyuavcan.application + + asyncio.get_running_loop().slow_callback_duration = 3.0 + _ = generated_packages + + first_run, run_config = parameters + if first_run: + # At the first run, force the demo script to regenerate packages. + # The following runs shall not force this behavior to save time and enhance branch coverage. + print("FORCE DSDL RECOMPILATION") + dsdl_output_path = pathlib.Path(tempfile.gettempdir(), "dsdl-for-my-program") + if dsdl_output_path.exists(): + shutil.rmtree(dsdl_output_path) + + # The demo may need to generate packages as well, so we launch it first. + demo_proc_env_vars = run_config.demo_env_vars.copy() + demo_proc_env_vars.update( + { + "PYUAVCAN_LOGLEVEL": "INFO", + "PATH": os.environ.get("PATH", ""), + "SYSTEMROOT": os.environ.get("SYSTEMROOT", ""), # https://github.com/appveyor/ci/issues/1995 + } + ) + demo_proc = BackgroundChildProcess( + "python", + "-m", + "coverage", + "run", + str(DEMO_DIR / "demo_app.py"), + environment_variables=demo_proc_env_vars, + ) + assert demo_proc.alive + print("DEMO APP STARTED WITH PID", demo_proc.pid, "FROM", pathlib.Path.cwd()) + + # Initialize the local node for testing. + try: + transport = run_config.local_transport_factory(123) # type: ignore + presentation = pyuavcan.presentation.Presentation(transport) + except Exception: + demo_proc.kill() + raise + + # Run the test and make sure to clean up at exit to avoid resource usage warnings in the test logs. + try: + local_node_info = uavcan.node.GetInfo_1_0.Response( + protocol_version=uavcan.node.Version_1_0(*pyuavcan.UAVCAN_SPECIFICATION_VERSION), + software_version=uavcan.node.Version_1_0(*pyuavcan.__version_info__[:2]), + name="org.uavcan.pyuavcan.test.demo_app", + ) + node = pyuavcan.application.Node(presentation, local_node_info, with_diagnostic_subscriber=True) + + # Construct the ports we will be using to interact with the demo application. + sub_heartbeat = node.presentation.make_subscriber_with_fixed_subject_id(uavcan.node.Heartbeat_1_0) + sub_diagnostics = node.presentation.make_subscriber_with_fixed_subject_id(uavcan.diagnostic.Record_1_1) + pub_temperature = node.presentation.make_publisher(uavcan.si.sample.temperature.Scalar_1_0, 2345) + client_get_info = node.presentation.make_client_with_fixed_service_id(uavcan.node.GetInfo_1_0, DEMO_APP_NODE_ID) + client_command = node.presentation.make_client_with_fixed_service_id( + uavcan.node.ExecuteCommand_1_1, DEMO_APP_NODE_ID + ) + client_least_squares = node.presentation.make_client( + sirius_cyber_corp.PerformLinearLeastSquaresFit_1_0, 123, DEMO_APP_NODE_ID + ) + + # At the first run, the usage demo might take a long time to start because it has to compile DSDL. + # That's why we wait for it here to announce readiness by subscribing to the heartbeat. + assert demo_proc.alive + first_hb_transfer = await sub_heartbeat.receive_for(100.0) # Pick a sensible start-up timeout. + print("FIRST HEARTBEAT:", first_hb_transfer) + assert first_hb_transfer + assert first_hb_transfer[1].source_node_id == DEMO_APP_NODE_ID + assert first_hb_transfer[1].transfer_id < 10 # We may have missed a couple but not too many! + assert demo_proc.alive + # Once the heartbeat is in, we know that the demo is ready for being tested. + + # Validate GetInfo. + client_get_info.priority = pyuavcan.transport.Priority.EXCEPTIONAL + client_get_info.transfer_id_counter.override(22) + info_transfer = await client_get_info.call(uavcan.node.GetInfo_1_0.Request()) + print("GET INFO RESPONSE:", info_transfer) + assert info_transfer + info, transfer = info_transfer + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert transfer.transfer_id == 22 + assert transfer.priority == pyuavcan.transport.Priority.EXCEPTIONAL + assert isinstance(info, uavcan.node.GetInfo_1_0.Response) + assert info.name.tobytes().decode() == "org.uavcan.pyuavcan.demo.demo_app" + assert info.protocol_version.major == pyuavcan.UAVCAN_SPECIFICATION_VERSION[0] + assert info.protocol_version.minor == pyuavcan.UAVCAN_SPECIFICATION_VERSION[1] + assert info.software_version.major == 1 + assert info.software_version.minor == 0 + + # Test the linear regression service. + solution_transfer = await client_least_squares.call( + sirius_cyber_corp.PerformLinearLeastSquaresFit_1_0.Request( + points=[ + sirius_cyber_corp.PointXY_1_0(x=1, y=2), + sirius_cyber_corp.PointXY_1_0(x=10, y=20), + ] + ) + ) + print("LINEAR REGRESSION RESPONSE:", info_transfer) + assert solution_transfer + solution, transfer = solution_transfer + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert transfer.transfer_id == 0 + assert transfer.priority == pyuavcan.transport.Priority.NOMINAL + assert isinstance(solution, sirius_cyber_corp.PerformLinearLeastSquaresFit_1_0.Response) + assert solution.slope == pytest.approx(2.0) + assert solution.y_intercept == pytest.approx(0.0) + + # Publish temperature. The result will be validated later. + pub_temperature.priority = pyuavcan.transport.Priority.SLOW + pub_temperature.publish_soon(uavcan.si.sample.temperature.Scalar_1_0(kelvin=321.5)) + + # Test the command execution service. + # Bad command. + result_transfer = await client_command.call( + uavcan.node.ExecuteCommand_1_1.Request( + command=uavcan.node.ExecuteCommand_1_1.Request.COMMAND_STORE_PERSISTENT_STATES + ) + ) + print("BAD COMMAND RESPONSE:", info_transfer) + assert result_transfer + result, transfer = result_transfer + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert transfer.transfer_id == 0 + assert transfer.priority == pyuavcan.transport.Priority.NOMINAL + assert isinstance(result, uavcan.node.ExecuteCommand_1_1.Response) + assert result.status == result.STATUS_BAD_COMMAND + # Good custom command. + result_transfer = await client_command.call( + uavcan.node.ExecuteCommand_1_1.Request( + command=23456, + parameter="This is my custom command parameter", + ) + ) + print("CUSTOM COMMAND RESPONSE:", info_transfer) + assert result_transfer + result, transfer = result_transfer + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert transfer.transfer_id == 1 + assert transfer.priority == pyuavcan.transport.Priority.NOMINAL + assert isinstance(result, uavcan.node.ExecuteCommand_1_1.Response) + assert result.status == result.STATUS_SUCCESS + # FINAL COMMAND: ASK THE NODE TO TERMINATE ITSELF. + assert demo_proc.alive, "Can't ask a dead node to kill itself, it's impolite." + result_transfer = await client_command.call( + uavcan.node.ExecuteCommand_1_1.Request(command=uavcan.node.ExecuteCommand_1_1.Request.COMMAND_POWER_OFF) + ) + print("POWER OFF COMMAND RESPONSE:", info_transfer) + assert result_transfer + result, transfer = result_transfer + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert transfer.transfer_id == 2 + assert transfer.priority == pyuavcan.transport.Priority.NOMINAL + assert isinstance(result, uavcan.node.ExecuteCommand_1_1.Response) + assert result.status == result.STATUS_SUCCESS + + # Validate the heartbeats (all of them) while waiting for the node to terminate. + prev_hb_transfer = first_hb_transfer + num_heartbeats = 0 + while True: + # The timeout will get triggered at some point because the demo app has been asked to stop. + hb_transfer = await sub_heartbeat.receive_for(1.0) + if hb_transfer is None: + break + hb, transfer = hb_transfer + assert num_heartbeats <= transfer.transfer_id <= 300 + assert transfer.priority == pyuavcan.transport.Priority.NOMINAL + assert transfer.source_node_id == DEMO_APP_NODE_ID + assert hb.health.value == hb.health.NOMINAL + assert hb.mode.value == hb.mode.OPERATIONAL + assert num_heartbeats <= hb.uptime <= 300 + assert hb.uptime == prev_hb_transfer[0].uptime + 1 + assert transfer.transfer_id == prev_hb_transfer[1].transfer_id + 1 + prev_hb_transfer = hb_transfer + num_heartbeats += 1 + assert num_heartbeats > 0 + + # Validate the diagnostic messages while waiting for the node to terminate. + async def get_next_diagnostic() -> typing.Optional[str]: + d = await sub_diagnostics.receive_for(1.0) + if d: + print("RECEIVED DIAGNOSTIC:", d) + m, t = d + assert t.source_node_id == DEMO_APP_NODE_ID + assert t.priority == pyuavcan.transport.Priority.OPTIONAL + assert isinstance(m, uavcan.diagnostic.Record_1_1) + s = m.text.tobytes().decode() + assert isinstance(s, str) + return s + return None + + assert re.match(rf"Least squares request from {transport.local_node_id}.*", await get_next_diagnostic() or "") + assert re.match(r"Solution for .*: ", await get_next_diagnostic() or "") + assert re.match(rf"Temperature .* from {transport.local_node_id}.*", await get_next_diagnostic() or "") + assert not await get_next_diagnostic() + + # We've asked the node to terminate, wait for it here. + out_demo_proc = demo_proc.wait(10.0)[1].splitlines() + print("DEMO APP FINAL OUTPUT:", *out_demo_proc, sep="\n\t") + assert out_demo_proc + assert any(re.match(r"TEMPERATURE \d+\.\d+ C", s) for s in out_demo_proc) + assert any(re.match(r"CUSTOM COMMAND PARAMETER: This is my custom command parameter", s) for s in out_demo_proc) + finally: + presentation.close() + demo_proc.kill() + await asyncio.sleep(2.0) # Let coroutines terminate properly to avoid resource usage warnings. diff --git a/tests/cli/_subprocess.py b/tests/demo/_subprocess.py similarity index 64% rename from tests/cli/_subprocess.py rename to tests/demo/_subprocess.py index a2abe23d3..042ca6846 100644 --- a/tests/cli/_subprocess.py +++ b/tests/demo/_subprocess.py @@ -1,69 +1,18 @@ -# Copyright (c) 2019 UAVCAN Consortium +# Copyright (c) 2020 UAVCAN Consortium # This software is distributed under the terms of the MIT License. # Author: Pavel Kirienko from __future__ import annotations -import os import sys import shutil import typing import logging -import pathlib import subprocess -from subprocess import CalledProcessError as CalledProcessError - - -DEMO_DIR = pathlib.Path(__file__).absolute().parent.parent / "demo" _logger = logging.getLogger(__name__) -def run_process( - *args: str, - timeout: typing.Optional[float] = None, - environment_variables: typing.Optional[typing.Dict[str, str]] = None, -) -> str: - r""" - This is a wrapper over :func:`subprocess.check_output`. - It adds all directories containing runnable scripts (the CLI tool and the demos) to PATH to make them invokable. - - :param args: The args to run. - - :param timeout: Give up waiting if the command could not be completed in this much time and raise TimeoutExpired. - No limit by default. - - :param environment_variables: appends or overrides environment variables for the process. - - :return: stdout of the command. - - >>> run_process('ping', '127.0.0.1', timeout=0.1) - Traceback (most recent call last): - ... - subprocess.TimeoutExpired: ... - """ - cmd = _make_process_args(*args) - _logger.info("Running process with timeout=%s: %s", timeout if timeout is not None else "inf", " ".join(cmd)) - env = _get_env() - if environment_variables: - env.update(environment_variables) - # Can't use shell=True with timeout; see https://stackoverflow.com/questions/36952245/subprocess-timeout-failure - stdout = subprocess.check_output(cmd, stderr=sys.stderr, timeout=timeout, encoding="utf8", env=env) - assert isinstance(stdout, str) - return stdout - - -def run_cli_tool( - *args: str, - timeout: typing.Optional[float] = None, - environment_variables: typing.Optional[typing.Dict[str, str]] = None, -) -> str: - """ - A wrapper over :func:`run_process` that runs the CLI tool with the specified arguments. - """ - return run_process("python", "-m", "pyuavcan", *args, timeout=timeout, environment_variables=environment_variables) - - class BackgroundChildProcess: r""" A wrapper over :class:`subprocess.Popen`. @@ -85,12 +34,12 @@ class BackgroundChildProcess: def __init__(self, *args: str, environment_variables: typing.Optional[typing.Dict[str, str]] = None): cmd = _make_process_args(*args) - _logger.info("Starting background child process: %s", " ".join(cmd)) + _logger.info("Starting in background: %s with env vars: %s", args, environment_variables) - try: # Windows-specific. + if sys.platform.startswith("win"): # If the current process group is used, CTRL_C_EVENT will kill the parent and everyone in the group! - creationflags: int = subprocess.CREATE_NEW_PROCESS_GROUP # type: ignore - except AttributeError: # Not on Windows. + creationflags: int = subprocess.CREATE_NEW_PROCESS_GROUP + else: creationflags = 0 # Buffering must be DISABLED, otherwise we can't read data on Windows after the process is interrupted. @@ -139,16 +88,13 @@ def pid(self) -> int: def alive(self) -> bool: return self._inferior.poll() is None - def __del__(self) -> None: - if self._inferior.poll() is None: - self._inferior.kill() - def _get_env(environment_variables: typing.Optional[typing.Dict[str, str]] = None) -> typing.Dict[str, str]: - env = os.environ.copy() # Buffering must be DISABLED, otherwise we can't read data on Windows after the process is interrupted. # For some reason stdout is not flushed at exit there. - env["PYTHONUNBUFFERED"] = "1" + env = { + "PYTHONUNBUFFERED": "1", + } env.update(environment_variables or {}) return env diff --git a/tests/dsdl/_compiler.py b/tests/dsdl/_compiler.py index e17b4f8cb..185585e13 100644 --- a/tests/dsdl/_compiler.py +++ b/tests/dsdl/_compiler.py @@ -8,31 +8,34 @@ import tempfile import pytest import pyuavcan.dsdl -from .conftest import TEST_DATA_TYPES_DIR, PUBLIC_REGULATED_DATA_TYPES_DIR +from .conftest import DEMO_DIR def _unittest_bad_usage() -> None: with pytest.raises(TypeError): - pyuavcan.dsdl.generate_package(TEST_DATA_TYPES_DIR, TEST_DATA_TYPES_DIR) # type: ignore + # noinspection PyTypeChecker + pyuavcan.dsdl.generate_package("irrelevant", "irrelevant") # type: ignore def _unittest_module_import_path_usage_suggestion(caplog: typing.Any) -> None: - caplog.set_level(logging.WARNING) + caplog.set_level(logging.INFO) with tempfile.TemporaryDirectory() as output_directory: output_directory_name = pathlib.Path(output_directory).resolve() caplog.clear() pyuavcan.dsdl.generate_package( - PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan", + DEMO_DIR / "public_regulated_data_types" / "uavcan", output_directory=output_directory, ) logs = caplog.record_tuples - assert len(logs) == 1 - print("Captured warning log entry:", logs[0], sep="\n") - assert "dsdl" in logs[0][0] - assert logs[0][1] == logging.WARNING - assert " path" in logs[0][2] - assert "Path(" not in logs[0][2] # Ensure decent formatting - assert str(output_directory_name) in logs[0][2] + print("Captured log entries:", logs, sep="\n") + for e in logs: + if "dsdl" in e[0] and str(output_directory_name) in e[2]: + assert e[1] == logging.INFO + assert " path" in e[2] + assert "Path(" not in e[2] # Ensure decent formatting + break + else: + assert False def _unittest_issue_133() -> None: diff --git a/tests/dsdl/conftest.py b/tests/dsdl/conftest.py index ede720ee2..f957bdbcb 100644 --- a/tests/dsdl/conftest.py +++ b/tests/dsdl/conftest.py @@ -19,9 +19,8 @@ # Please maintain these carefully if you're changing the project's directory structure. TEST_ROOT_DIR = pathlib.Path(__file__).parent.parent LIBRARY_ROOT_DIR = TEST_ROOT_DIR.parent -DESTINATION_DIR = LIBRARY_ROOT_DIR / pathlib.Path(".test_dsdl_generated") -PUBLIC_REGULATED_DATA_TYPES_DIR = TEST_ROOT_DIR / "public_regulated_data_types" -TEST_DATA_TYPES_DIR = pathlib.Path(__file__).parent / "namespaces" +DEMO_DIR = LIBRARY_ROOT_DIR / "demo" +DESTINATION_DIR = LIBRARY_ROOT_DIR / ".test_dsdl_generated" _CACHE_FILE_NAME = "pydsdl_cache.pickle.tmp" @@ -69,18 +68,18 @@ def generate_packages() -> typing.List[pyuavcan.dsdl.GeneratedPackageInfo]: pydsdl_logger.setLevel(logging.INFO) out = [ pyuavcan.dsdl.generate_package( - PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan", + DEMO_DIR / "public_regulated_data_types" / "uavcan", [], DESTINATION_DIR, ), pyuavcan.dsdl.generate_package( - TEST_DATA_TYPES_DIR / "test_dsdl_namespace", - [PUBLIC_REGULATED_DATA_TYPES_DIR / "uavcan"], + DEMO_DIR / "custom_data_types" / "sirius_cyber_corp", + [], DESTINATION_DIR, ), pyuavcan.dsdl.generate_package( - TEST_DATA_TYPES_DIR / "sirius_cyber_corp", - [], + TEST_ROOT_DIR / "dsdl" / "test_dsdl_namespace", + [DEMO_DIR / "public_regulated_data_types" / "uavcan"], DESTINATION_DIR, ), ] diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/A.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/A.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/A.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/A.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/A.1.1.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/A.1.1.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/A.1.1.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/A.1.1.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/BDelimited.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/BDelimited.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/BDelimited.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/BDelimited.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/BDelimited.1.1.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/BDelimited.1.1.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/BDelimited.1.1.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/BDelimited.1.1.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/BSealed.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/BSealed.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/BSealed.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/BSealed.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/CFixed.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/CFixed.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/CFixed.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/CFixed.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/CFixed.1.1.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/CFixed.1.1.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/CFixed.1.1.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/CFixed.1.1.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/CVariable.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/CVariable.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/CVariable.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/CVariable.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/delimited/CVariable.1.1.uavcan b/tests/dsdl/test_dsdl_namespace/delimited/CVariable.1.1.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/delimited/CVariable.1.1.uavcan rename to tests/dsdl/test_dsdl_namespace/delimited/CVariable.1.1.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/if/B.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/if/B.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/if/B.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/if/B.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/if/C.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/if/C.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/if/C.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/if/C.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/if/del.1.0.uavcan b/tests/dsdl/test_dsdl_namespace/if/del.1.0.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/if/del.1.0.uavcan rename to tests/dsdl/test_dsdl_namespace/if/del.1.0.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/numpy/Complex.254.255.uavcan b/tests/dsdl/test_dsdl_namespace/numpy/Complex.254.255.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/numpy/Complex.254.255.uavcan rename to tests/dsdl/test_dsdl_namespace/numpy/Complex.254.255.uavcan diff --git a/tests/dsdl/namespaces/test_dsdl_namespace/numpy/RGB888_3840x2748.0.1.uavcan b/tests/dsdl/test_dsdl_namespace/numpy/RGB888_3840x2748.0.1.uavcan similarity index 100% rename from tests/dsdl/namespaces/test_dsdl_namespace/numpy/RGB888_3840x2748.0.1.uavcan rename to tests/dsdl/test_dsdl_namespace/numpy/RGB888_3840x2748.0.1.uavcan diff --git a/tests/transport/can/_can.py b/tests/transport/can/_can.py index c9805fda9..24cac0e51 100644 --- a/tests/transport/can/_can.py +++ b/tests/transport/can/_can.py @@ -81,20 +81,6 @@ async def _unittest_can_transport_anon() -> None: InputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), None), meta ) - server_responder = tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.RESPONSE), 123), meta - ) - assert server_responder is tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.RESPONSE), 123), meta - ) - - client_requester = tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 123), meta - ) - assert client_requester is tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 123), meta - ) - client_listener = tr.get_input_session( InputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.RESPONSE), 123), meta ) @@ -115,7 +101,7 @@ async def _unittest_can_transport_anon() -> None: del inputs print("OUTPUTS:", tr.output_sessions) - assert set(tr.output_sessions) == {broadcaster, server_responder, client_requester} + assert set(tr.output_sessions) == {broadcaster} # # Basic exchange test, no one is listening @@ -170,11 +156,13 @@ def validate_timestamp(timestamp: Timestamp) -> None: # Can't send anonymous service transfers with pytest.raises(OperationNotDefinedForAnonymousNodeError): - assert await client_requester.send( - Transfer(timestamp=ts, priority=Priority.IMMEDIATE, transfer_id=0, fragmented_payload=[]), - tr.loop.time() + 1.0, + tr.get_output_session( + OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.RESPONSE), 123), meta + ) + with pytest.raises(OperationNotDefinedForAnonymousNodeError): + tr.get_output_session( + OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 123), meta ) - assert client_requester.sample_statistics() == SessionStatistics() # Not incremented! # Can't send multiframe anonymous messages with pytest.raises(OperationNotDefinedForAnonymousNodeError): @@ -632,10 +620,10 @@ def validate_timestamp(timestamp: Timestamp) -> None: transfer_id=12, fragmented_payload=[ _mem( - "Until philosophers are kings, or the kings and princes of this world have the spirit and power " - "of philosophy, and political greatness and wisdom meet in one, and those commoner natures who " - "pursue either to the exclusion of the other are compelled to stand aside, cities will never " - "have rest from their evils " + "Until philosophers are kings, or the kings and princes of this world have the spirit and " + "power of philosophy, and political greatness and wisdom meet in one, and those commoner " + "natures who pursue either to the exclusion of the other are compelled to stand aside, " + "cities will never have rest from their evils " ), _mem("- no, nor the human race, as I believe - "), _mem("and then only will this our State have a possibility of life and behold the light of day."), diff --git a/tests/transport/redundant/_redundant.py b/tests/transport/redundant/_redundant.py index c5d55c31f..5e9c39e3e 100644 --- a/tests/transport/redundant/_redundant.py +++ b/tests/transport/redundant/_redundant.py @@ -122,7 +122,9 @@ async def _unittest_redundant_transport(caplog: typing.Any) -> None: ) assert tr_a.local_node_id == 111 assert ( - repr(tr_a) == "RedundantTransport(LoopbackTransport(local_node_id=111), LoopbackTransport(local_node_id=111))" + repr(tr_a) + == "RedundantTransport(LoopbackTransport(local_node_id=111, allow_anonymous_transfers=True)," + + " LoopbackTransport(local_node_id=111, allow_anonymous_transfers=True))" ) assert await pub_a.send( @@ -190,7 +192,9 @@ async def _unittest_redundant_transport(caplog: typing.Any) -> None: assert tr_a.protocol_parameters == cyc_proto_params, "Protocol parameter mismatch" assert tr_a.local_node_id == 111 assert ( - repr(tr_a) == "RedundantTransport(LoopbackTransport(local_node_id=111), LoopbackTransport(local_node_id=111))" + repr(tr_a) + == "RedundantTransport(LoopbackTransport(local_node_id=111, allow_anonymous_transfers=True)," + + " LoopbackTransport(local_node_id=111, allow_anonymous_transfers=True))" ) # Exchange test. @@ -313,3 +317,19 @@ def mon(_x: object) -> None: tr.attach_inferior(inf_b) assert inf_a.capture_handlers == [mon] assert inf_b.capture_handlers == [mon] + + +def _unittest_redundant_transport_reconfiguration() -> None: + from pyuavcan.transport import OutputSessionSpecifier, MessageDataSpecifier, PayloadMetadata + + tr = RedundantTransport() + tr.attach_inferior(LoopbackTransport(1234)) + ses = tr.get_output_session(OutputSessionSpecifier(MessageDataSpecifier(5555), None), PayloadMetadata(0)) + assert ses + tr.detach_inferior(tr.inferiors[0]) + tr.attach_inferior(LoopbackTransport(1235)) # Different node-ID + tr.detach_inferior(tr.inferiors[0]) + tr.attach_inferior(LoopbackTransport(None, allow_anonymous_transfers=True)) # Anonymous + with pytest.raises(pyuavcan.transport.OperationNotDefinedForAnonymousNodeError): + tr.attach_inferior(LoopbackTransport(None, allow_anonymous_transfers=False)) + assert len(tr.inferiors) == 1 diff --git a/tests/transport/serial/_serial.py b/tests/transport/serial/_serial.py index e941f0bb2..141bef41c 100644 --- a/tests/transport/serial/_serial.py +++ b/tests/transport/serial/_serial.py @@ -80,13 +80,6 @@ async def _unittest_serial_transport(caplog: typing.Any) -> None: InputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), None), meta ) - client_requester = tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 3210), meta - ) - assert client_requester is tr.get_output_session( - OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 3210), meta - ) - client_listener = tr.get_input_session( InputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.RESPONSE), 3210), meta ) @@ -97,7 +90,7 @@ async def _unittest_serial_transport(caplog: typing.Any) -> None: print("INPUTS:", tr.input_sessions) print("OUTPUTS:", tr.output_sessions) assert set(tr.input_sessions) == {subscriber_promiscuous, subscriber_selective, server_listener, client_listener} - assert set(tr.output_sessions) == {broadcaster, client_requester} + assert set(tr.output_sessions) == {broadcaster} assert tr.sample_statistics() == SerialTransportStatistics() # @@ -145,11 +138,8 @@ async def _unittest_serial_transport(caplog: typing.Any) -> None: # with pytest.raises(pyuavcan.transport.OperationNotDefinedForAnonymousNodeError): # Anonymous nodes can't emit service transfers. - assert await client_requester.send( - Transfer( - timestamp=Timestamp.now(), priority=Priority.HIGH, transfer_id=88888, fragmented_payload=payload_single - ), - monotonic_deadline=get_monotonic() + 5.0, + tr.get_output_session( + OutputSessionSpecifier(ServiceDataSpecifier(333, ServiceDataSpecifier.Role.REQUEST), 3210), meta ) #