Add end-to-end tests to test CLI output

Ref. eng/recordflux/RecordFlux#860

Makefile

@@ -131,11 +131,11 @@ check_doc:
 	poetry run tools/check_grammar.py --document doc/language_reference/language_reference.rst --verbal-map doc/language_reference/verbal_mapping.json examples/specs/*.rflx examples/apps/*/specs/*.rflx tests/data/specs/*.rflx tests/data/specs/parse_only/*.rflx
 	poetry run tools/check_grammar.py --invalid --document doc/language_reference/language_reference.rst --verbal-map doc/language_reference/verbal_mapping.json tests/data/specs/invalid/{incorrect_comment_only,incorrect_empty_file,incorrect_specification}.rflx
 
-.PHONY: test test_rflx test_examples test_coverage test_unit_coverage test_property test_tools test_ide test_optimized test_compilation test_binary_size test_installation test_specs test_apps
+.PHONY: test test_rflx test_examples test_coverage test_unit_coverage test_language_coverage test_end_to_end test_property test_tools test_ide test_optimized test_compilation test_binary_size test_installation test_specs test_apps
 
 test: test_rflx test_examples
 
-test_rflx: test_coverage test_unit_coverage test_language_coverage test_property test_tools test_ide test_optimized test_compilation test_binary_size test_installation
+test_rflx: test_coverage test_unit_coverage test_language_coverage test_end_to_end test_property test_tools test_ide test_optimized test_compilation test_binary_size test_installation
 
 test_examples: test_specs test_apps
 
@@ -154,6 +154,9 @@ test_unit_coverage:
 test_language_coverage:
 	timeout -k 60 7200 $(PYTEST) --cov=rflx/lang --cov-branch --cov-fail-under=73.8 --cov-report=term-missing:skip-covered tests/language
 
+test_end_to_end:
+	$(PYTEST) tests/end-to-end
+
 test_property:
 	$(PYTEST) tests/property
 

tests/README.md

@@ -29,6 +29,10 @@ The executability and provability tests require the definition of a session call
 
 Session functions can be defined by putting a custom implementation of the `Session` package inside the `src` directory.
 
+### End-to-End Tests (`tests/end-to-end`)
+
+End-to-end tests verify the functionality of the entire application. These tests ensure that the application behaves as intended, particularly focusing on its primary use cases.
+
 ### Property Tests (`tests/property`)
 
 Property-based testing based on [Hypothesis](https://hypothesis.readthedocs.io/).

tests/const.py

@@ -8,6 +8,7 @@
 IDE_DIR = TEST_DIR / "ide"
 GENERATED_DIR = TEST_DIR / "spark" / "generated"
 FEATURE_DIR = TEST_DIR / "feature"
+VALIDATOR_DIR = DATA_DIR / "validator"
 
 EX_SPEC_DIR = Path("examples/specs")
 MAIN = "main.adb"

tests/end-to-end/cli_test.py

@@ -0,0 +1,215 @@
+import subprocess
+import textwrap
+from pathlib import Path
+
+from tests.const import DATA_DIR, SPEC_DIR, VALIDATOR_DIR
+
+
+def test_check() -> None:
+    p = subprocess.run(
+        ["rflx", "--no-caching", "check", SPEC_DIR / "ethernet.rflx"],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == ""
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        """\
+        Parsing tests/data/specs/ethernet.rflx
+        Processing Ethernet
+        Verifying __BUILTINS__::Boolean
+        Verifying __INTERNAL__::Opaque
+        Verifying Ethernet::Address
+        Verifying Ethernet::Type_Length
+        Verifying Ethernet::TPID
+        Verifying Ethernet::TCI
+        Verifying Ethernet::Frame
+        """,
+    )
+
+
+def test_check_no_verification() -> None:
+    p = subprocess.run(
+        [
+            "rflx",
+            "--no-caching",
+            "--unsafe",
+            "--no-verification",
+            "check",
+            SPEC_DIR / "ethernet.rflx",
+        ],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == "model: warning: model verification skipped\n"
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        """\
+        Parsing tests/data/specs/ethernet.rflx
+        Processing Ethernet
+        """,
+    )
+
+
+def test_generate(tmp_path: Path) -> None:
+    p = subprocess.run(
+        [
+            "rflx",
+            "--no-caching",
+            "generate",
+            "-d",
+            tmp_path,
+            SPEC_DIR / "ethernet.rflx",
+        ],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == ""
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        f"""\
+        Parsing tests/data/specs/ethernet.rflx
+        Processing Ethernet
+        Verifying __BUILTINS__::Boolean
+        Verifying __INTERNAL__::Opaque
+        Verifying Ethernet::Address
+        Verifying Ethernet::Type_Length
+        Verifying Ethernet::TPID
+        Verifying Ethernet::TCI
+        Verifying Ethernet::Frame
+        Generating Ethernet::Address
+        Generating Ethernet::Type_Length
+        Generating Ethernet::TPID
+        Generating Ethernet::TCI
+        Generating Ethernet::Frame
+        Creating {tmp_path}/rflx-ethernet.ads
+        Creating {tmp_path}/rflx-ethernet-frame.ads
+        Creating {tmp_path}/rflx-ethernet-frame.adb
+        Creating {tmp_path}/rflx-rflx_arithmetic.ads
+        Creating {tmp_path}/rflx-rflx_builtin_types-conversions.ads
+        Creating {tmp_path}/rflx-rflx_builtin_types.ads
+        Creating {tmp_path}/rflx-rflx_generic_types.ads
+        Creating {tmp_path}/rflx-rflx_generic_types-generic_operators.ads
+        Creating {tmp_path}/rflx-rflx_generic_types-generic_operations.ads
+        Creating {tmp_path}/rflx-rflx_message_sequence.ads
+        Creating {tmp_path}/rflx-rflx_scalar_sequence.ads
+        Creating {tmp_path}/rflx-rflx_types.ads
+        Creating {tmp_path}/rflx-rflx_types-operators.ads
+        Creating {tmp_path}/rflx-rflx_types-operations.ads
+        Creating {tmp_path}/rflx-rflx_arithmetic.adb
+        Creating {tmp_path}/rflx-rflx_generic_types-generic_operations.adb
+        Creating {tmp_path}/rflx-rflx_message_sequence.adb
+        Creating {tmp_path}/rflx-rflx_scalar_sequence.adb
+        Creating {tmp_path}/rflx.ads
+        """,
+    )
+    assert (tmp_path / "rflx.ads").is_file()
+
+
+def test_graph(tmp_path: Path) -> None:
+    p = subprocess.run(
+        [
+            "rflx",
+            "--no-caching",
+            "graph",
+            "-d",
+            tmp_path,
+            SPEC_DIR / "ethernet.rflx",
+        ],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == ""
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        f"""\
+        Parsing tests/data/specs/ethernet.rflx
+        Processing Ethernet
+        Verifying __BUILTINS__::Boolean
+        Verifying __INTERNAL__::Opaque
+        Verifying Ethernet::Address
+        Verifying Ethernet::Type_Length
+        Verifying Ethernet::TPID
+        Verifying Ethernet::TCI
+        Verifying Ethernet::Frame
+        Creating {tmp_path}/Ethernet_Frame.svg
+        """,
+    )
+    assert (tmp_path / "Ethernet_Frame.svg").is_file()
+
+
+def test_validate() -> None:
+    p = subprocess.run(
+        [
+            "rflx",
+            "--no-caching",
+            "validate",
+            "-v",
+            VALIDATOR_DIR / "ethernet" / "frame" / "valid",
+            SPEC_DIR / "ethernet.rflx",
+            "Ethernet::Frame",
+        ],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == textwrap.dedent(
+        """\
+        tests/data/validator/ethernet/frame/valid/802.3-LLC-CDP.raw                      PASSED
+        tests/data/validator/ethernet/frame/valid/EII-802.1AD-802.1Q-IPv4.raw            PASSED
+        tests/data/validator/ethernet/frame/valid/EII-802.1Q-802.1Q-IPv4-ICMP.raw        PASSED
+        tests/data/validator/ethernet/frame/valid/EII-802.1Q-LLC-CDP.raw                 PASSED
+        tests/data/validator/ethernet/frame/valid/EII-802.1Q-LLC-STP.raw                 PASSED
+        tests/data/validator/ethernet/frame/valid/ethernet_802.3.raw                     PASSED
+        tests/data/validator/ethernet/frame/valid/ethernet_double_vlan_tag.raw           PASSED
+        tests/data/validator/ethernet/frame/valid/ethernet_ipv4_udp.raw                  PASSED
+        tests/data/validator/ethernet/frame/valid/ethernet_vlan_tag.raw                  PASSED
+        """,
+    )
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        """\
+        Parsing tests/data/specs/ethernet.rflx
+        Processing Ethernet
+        Verifying __BUILTINS__::Boolean
+        Verifying __INTERNAL__::Opaque
+        Verifying Ethernet::Address
+        Verifying Ethernet::Type_Length
+        Verifying Ethernet::TPID
+        Verifying Ethernet::TCI
+        Verifying Ethernet::Frame
+        """,
+    )
+
+
+def test_install(tmp_path: Path) -> None:
+    p = subprocess.run(
+        ["rflx", "install", "gnatstudio", "--gnat-studio-dir", tmp_path],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == textwrap.dedent(
+        f"""\
+        Installing RecordFlux plugin into "{tmp_path}/plug-ins"
+        """,
+    )
+    assert p.stderr.decode("utf-8") == ""
+    assert (tmp_path / "plug-ins" / "recordflux.py").is_file()
+
+
+def test_convert(tmp_path: Path) -> None:
+    p = subprocess.run(
+        ["rflx", "convert", "iana", "-d", tmp_path, DATA_DIR / "bootp-dhcp-parameters.xml"],
+        capture_output=True,
+    )
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == ""
+    assert p.stderr.decode("utf-8") == ""
+    assert (tmp_path / "bootp_dhcp_parameters.rflx").is_file()
+
+
+def test_run_ls() -> None:
+    p = subprocess.run(["rflx", "run_ls"], capture_output=True, input="")
+    assert p.returncode == 0
+    assert p.stdout.decode("utf-8") == ""
+    assert p.stderr.decode("utf-8") == textwrap.dedent(
+        """\
+        Starting IO server
+        Shutting down the server
+        Closing the event loop.
+        """,
+    )