diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md index 07759f5bbd..18f806d8fc 100644 --- a/docs/CHANGELOG.md +++ b/docs/CHANGELOG.md @@ -55,6 +55,7 @@ Test fixtures for use by clients are available for each release on the [Github r - ๐Ÿ”€ `ethereum_test_forks` forks now contain gas-calculating functions, which return the appropriate function to calculate the gas used by a transaction or memory function for the given fork ([#779](https://github.com/ethereum/execution-spec-tests/pull/779)). - ๐Ÿž Fix `Bytecode` class `__eq__` method ([#939](https://github.com/ethereum/execution-spec-tests/pull/939)). - ๐Ÿ”€ Update `pydantic` from 2.8.2 to 2.9.2 ([#960](https://github.com/ethereum/execution-spec-tests/pull/960)). +- โœจ Add the `et make test` command, an interactive CLI that helps users create a new test module and function ([#950](https://github.com/ethereum/execution-spec-tests/pull/950)). ### ๐Ÿ”ง EVM Tools diff --git a/docs/writing_tests/img/et_make_test.mp4 b/docs/writing_tests/img/et_make_test.mp4 new file mode 100644 index 0000000000..5555c93f9c Binary files /dev/null and b/docs/writing_tests/img/et_make_test.mp4 differ diff --git a/docs/writing_tests/index.md b/docs/writing_tests/index.md index 742e21e909..c1baca204a 100644 --- a/docs/writing_tests/index.md +++ b/docs/writing_tests/index.md @@ -1,5 +1,22 @@ # Writing Tests -The best way to get started is to use one of the existing test modules for inspiration. A good simple example is [tests.berlin.eip2930_access_list.test_acl.test_access_list](../tests/berlin/eip2930_access_list/test_acl/test_access_list.md). +The easiest way to get started is to use the interactive CLI: + +```console +uv run et make test +``` + +and modify the generated test module to suit your needs. + +
+ +
+ +For help deciding which test format to select, see [Types of Tests](./types_of_tests.md), in particular [Deciding on a Test Type](./types_of_tests.md#deciding-on-a-test-type). Otherwise, some simple test case examples to get started with are: + +- [tests.berlin.eip2930_access_list.test_acl.test_access_list](../tests/berlin/eip2930_access_list/test_acl/test_access_list.md). +- [tests.istanbul.eip1344_chainid.test_chainid.test_chainid](../tests/istanbul/eip1344_chainid/test_chainid/test_chainid.md). Please check that your code adheres to the repo's [Coding Standards](./code_standards.md) and read the other pages in this section for more background and an explanation of how to implement state transition and blockchain tests. diff --git a/docs/writing_tests/types_of_tests.md b/docs/writing_tests/types_of_tests.md index 4fedb814db..d0421fbf6d 100644 --- a/docs/writing_tests/types_of_tests.md +++ b/docs/writing_tests/types_of_tests.md @@ -58,3 +58,5 @@ def test_blob_type_tx_pre_fork( Whenever possible, use `state_test` to examine individual transactions. This method is more straightforward and less prone to external influences that can occur during block building. This provides more targeted testing since it does not invoke the client's block-building machinery. This reduces the risk of encountering false positives, particularly in exception scenarios (e.g., see issue [#343: "Zero max_fee_per_blob_gas test is ineffective"](https://github.com/ethereum/execution-spec-tests/issues/343)). + +Moreover, the `fill` command automatically additionally generates a `blockchain_test` for every `state_test` by wrapping the `state_test`'s transaction in a block. diff --git a/pyproject.toml b/pyproject.toml index 767de6a8a6..bf6c315e86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -107,6 +107,7 @@ order_fixtures = "cli.order_fixtures:order_fixtures" evm_bytes = "cli.evm_bytes:cli" hasher = "cli.hasher:main" env_init = "config.env:create_default_config" +et = "cli.et.cli:et" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/cli/et/__init__.py b/src/cli/et/__init__.py new file mode 100644 index 0000000000..3a70ec0339 --- /dev/null +++ b/src/cli/et/__init__.py @@ -0,0 +1,3 @@ +""" +This module is the entry point for the `et` command line interface. +""" diff --git a/src/cli/et/cli.py b/src/cli/et/cli.py new file mode 100644 index 0000000000..2cb05f4f7e --- /dev/null +++ b/src/cli/et/cli.py @@ -0,0 +1,29 @@ +""" +`et` is the dev CLI for EEST. It provides commands to help developers write tests. +Invoke using `uv run et`. +""" + +import click + +from cli.et.make.cli import make + + +@click.group() +def et(): + """ + `et` ๐Ÿ‘ฝ is the dev CLI for EEST. It provides commands to help developers write tests. + """ + pass + + +""" +################################ +|| || +|| Command Registration || +|| || +################################ + +Register nested commands here. For more information, see Click documentation: +https://click.palletsprojects.com/en/8.0.x/commands/#nested-handling-and-contexts +""" +et.add_command(make) diff --git a/src/cli/et/make/__init__.py b/src/cli/et/make/__init__.py new file mode 100644 index 0000000000..18ab0544ee --- /dev/null +++ b/src/cli/et/make/__init__.py @@ -0,0 +1,10 @@ +""" +Make CLI + +This module provides the `make` CLI command that helps you quickly scaffold +files. +""" + +from .cli import test + +__all__ = ["test"] diff --git a/src/cli/et/make/cli.py b/src/cli/et/make/cli.py new file mode 100644 index 0000000000..85065b2223 --- /dev/null +++ b/src/cli/et/make/cli.py @@ -0,0 +1,34 @@ +""" +The `make` CLI streamlines the process of scaffolding tasks, such as generating new test files, +enabling developers to concentrate on the core aspects of specification testing. + + +The module calls the appropriate function for the subcommand. If an invalid subcommand +is chosen, it throws an error and shows a list of valid subcommands. If no subcommand +is present, it shows a list of valid subcommands to choose from. +""" + +import click + +from .commands import test + + +@click.group() +def make(): + """ + Generate project files from the CLI. + """ + pass + + +""" +################################ +|| || +|| Command Registration || +|| || +################################ + +Register nested commands here. For more information, see Click documentation: +https://click.palletsprojects.com/en/8.0.x/commands/#nested-handling-and-contexts +""" +make.add_command(test) diff --git a/src/cli/et/make/commands/__init__.py b/src/cli/et/make/commands/__init__.py new file mode 100644 index 0000000000..f27406d62b --- /dev/null +++ b/src/cli/et/make/commands/__init__.py @@ -0,0 +1,9 @@ +""" +This subpackage holds subcommands for the make command. New subcommands must be created as +modules and exported from this package, then registered under the make command in +`cli.py`. +""" + +from .test import test + +__all__ = ["test"] diff --git a/src/cli/et/make/commands/test.py b/src/cli/et/make/commands/test.py new file mode 100644 index 0000000000..ebf115e632 --- /dev/null +++ b/src/cli/et/make/commands/test.py @@ -0,0 +1,117 @@ +""" +This module provides a CLI command to scaffold a test file. + +The `test` command guides the user through a series of prompts to generate a test file +based on the selected test type, fork, EIP number, and EIP name. The generated test file +is saved in the appropriate directory with a rendered template using Jinja2. +""" + +import os +import sys +from pathlib import Path + +import click +import jinja2 + +from cli.input import input_select, input_text +from config.docs import DocsConfig +from ethereum_test_forks import get_development_forks, get_forks + +template_loader = jinja2.PackageLoader("cli.et.make") +template_env = jinja2.Environment( + loader=template_loader, keep_trailing_newline=True, trim_blocks=True, lstrip_blocks=True +) + + +@click.command() +def test(): + """ + Create a new specification test file for an EIP. + + This function guides the user through a series of prompts to generate a test file + for Ethereum execution specifications. The user is prompted to select the type of test, + the fork to use, and to provide the EIP number and name. Based on the inputs, a test file + is created in the appropriate directory with a rendered template. + + Prompts: + - Choose the type of test to generate (State or Blockchain) + - Select the fork to use (Prague or Osaka) + - Enter the EIP number + - Enter the EIP name + + The generated test file is saved in the following format: + `tests/{fork}/eip{eip_number}_{eip_name}/test_{eip_name}.py` + + Example: + If the user selects "State" as the test type, "Prague" as the fork, + enters "1234" as the EIP number, + and "Sample EIP" as the EIP name, the generated file will be: + `tests/prague/eip1234_sample_eip/test_sample_eip.py` + + The function uses Jinja2 templates to render the content of the test file. + + Raises: + - FileNotFoundError: If the template file does not exist. + - IOError: If there is an error writing the file. + """ + test_type = input_select( + "Choose the type of test to generate", choices=["State", "Blockchain"] + ) + + fork_choices = [str(fork) for fork in get_forks()] + fork = input_select( + "Select the fork where this functionality was introduced", choices=fork_choices + ) + + eip_number = input_text("Enter the EIP number").strip() + + # TODO: Perhaps get the EIP name from the number using an API? + eip_name = input_text("Enter the EIP name").strip() + + test_name = eip_name.lower().replace(" ", "_") + + file_name = f"test_{test_name}.py" + + directory_path = Path("tests") / fork.lower() / f"eip{eip_number}_{test_name}" + + file_path = directory_path / file_name + + if file_path.exists(): + click.echo( + click.style(f"\n ๐Ÿ›‘ The target test module {file_path} already exists!", fg="red"), + err=True, + ) + sys.exit(1) + + # Create directories if they don't exist + os.makedirs(directory_path, exist_ok=True) + + template = template_env.get_template(f"{test_type.lower()}_test.py.j2") + rendered_template = template.render( + fork=fork, + eip_number=eip_number, + eip_name=eip_name, + test_name=test_name, + ) + + with open(file_path, "w") as file: + file.write(rendered_template) + + click.echo( + click.style( + f"\n ๐ŸŽ‰ Success! Test file created at: {file_path}", + fg="green", + ) + ) + + fork_option = "" + if fork in [dev_fork.name() for dev_fork in get_development_forks()]: + fork_option = f" --until={fork}" + + click.echo( + click.style( + f"\n ๐Ÿ“ Get started with tests: {DocsConfig().DOCS_URL__WRITING_TESTS}" + f"\n โ›ฝ To fill this test, run: `uv run fill {file_path}{fork_option}`", + fg="cyan", + ) + ) diff --git a/src/cli/et/make/templates/blockchain_test.py.j2 b/src/cli/et/make/templates/blockchain_test.py.j2 new file mode 100644 index 0000000000..c6d94629c9 --- /dev/null +++ b/src/cli/et/make/templates/blockchain_test.py.j2 @@ -0,0 +1,53 @@ +""" +A blockchain test for [EIP-{{eip_number}} {{eip_name}}](https://eips.ethereum.org/EIPS/eip-{{eip_number}}). +""" + +import pytest + +from ethereum_test_tools import Account, Alloc, Block, BlockchainTestFiller, Transaction +from ethereum_test_tools.vm.opcode import Opcodes as Op + +REFERENCE_SPEC_GIT_PATH = "DUMMY/eip-DUMMY.md" +REFERENCE_SPEC_VERSION = "DUMMY_VERSION" + +@pytest.mark.valid_from("{{fork}}") +def test_{{test_name}}(blockchain_test: BlockchainTestFiller, pre: Alloc): + """ + TODO: Enter a one-line test summary here. + + TODO: (Optional) Enter a more detailed test function description here. + """ + # TODO: Delete this explanation. + # In this demo test, the pre-state contains one EOA and one very simple + # smart contract. The EOA, `sender`, executes the smart contract, which + # simply sets the value of the contract's storage slot. + # The (non-exhaustive) post-state verifies that the storage slot was set + # correctly - this is checked when filling the test. + # + # One gotcha is ensuring that the transaction `gas_limit` is set high + # enough to cover the gas cost of the contract execution. + + storage_slot: int = 1 + + # TODO: Modify pre-state allocations here. + sender = pre.fund_eoa() + contract_address = pre.deploy_contract( + code=Op.SSTORE(storage_slot, 0x2) + Op.STOP, + storage={storage_slot: 0x1}, + ) + + tx = Transaction( + to=contract_address, + gas_limit=100000000, + data=b"", + value=0, + sender=sender, + {% if fork in ["Frontier", "Homestead"] %} + protected=False, + {% endif %} + ) + + # TODO: Modify post-state allocations here. + post = {contract_address: Account(storage={storage_slot: 0x2})} + + blockchain_test(pre=pre, blocks=[Block(txs=[tx])], post=post) diff --git a/src/cli/et/make/templates/state_test.py.j2 b/src/cli/et/make/templates/state_test.py.j2 new file mode 100644 index 0000000000..bcf25f789d --- /dev/null +++ b/src/cli/et/make/templates/state_test.py.j2 @@ -0,0 +1,55 @@ +""" +A state test for [EIP-{{eip_number}} {{eip_name}}](https://eips.ethereum.org/EIPS/eip-{{eip_number}}). +""" + +import pytest + +from ethereum_test_tools import Account, Alloc, Environment, StateTestFiller, Transaction +from ethereum_test_tools.vm.opcode import Opcodes as Op + +REFERENCE_SPEC_GIT_PATH = "DUMMY/eip-DUMMY.md" +REFERENCE_SPEC_VERSION = "DUMMY_VERSION" + +@pytest.mark.valid_from("{{fork}}") +def test_{{test_name}}(state_test: StateTestFiller, pre: Alloc): + """ + TODO: Enter a one-line test summary here. + + TODO: (Optional) Enter a more detailed test function description here. + """ + env = Environment() + + # TODO: Delete this explanation. + # In this demo test, the pre-state contains one EOA and one very simple + # smart contract. The EOA, `sender`, executes the smart contract, which + # simply sets the value of the contract's storage slot. + # The (non-exhaustive) post-state verifies that the storage slot was set + # correctly - this is checked when filling the test. + # + # One gotcha is ensuring that the transaction `gas_limit` is set high + # enough to cover the gas cost of the contract execution. + + storage_slot: int = 1 + + # TODO: Modify pre-state allocations here. + sender = pre.fund_eoa() + contract_address = pre.deploy_contract( + code=Op.SSTORE(storage_slot, 0x2) + Op.STOP, + storage={storage_slot: 0x1}, + ) + + tx = Transaction( + to=contract_address, + gas_limit=100000000, + data=b"", + value=0, + sender=sender, + {% if fork in ["Frontier", "Homestead"] %} + protected=False, + {% endif %} + ) + + # TODO: Modify post-state allocations here. + post = {contract_address: Account(storage={storage_slot: 0x2})} + + state_test(env=env, pre=pre, post=post, tx=tx) diff --git a/src/cli/et/quotes.py b/src/cli/et/quotes.py new file mode 100644 index 0000000000..9951c0a5fd --- /dev/null +++ b/src/cli/et/quotes.py @@ -0,0 +1,63 @@ +""" +This module contains a list of quotes related to system design. +""" + +import random +import textwrap + +make_something_great = [ + "๐ŸŽจ Simplicity is the ultimate sophistication. - Leonardo D.", + "๐Ÿ–Œ๏ธ Simplicity is an acquired taste. - Katharine G.", + "๐Ÿ’ก To create a memorable design you need to start with a thought thatโ€™s worth remembering." + " - Thomas M.", + "๐Ÿš€ Well begun is half done. - Aristotle", + "๐Ÿ–Œ๏ธ Designers are crazy and yet sane enough to know where to draw the line. - Benjamin W.", + "๐ŸŒŸ Creativity is piercing the mundane to find the marvelous. - Bill M.", + "๐Ÿ” Mistakes are the portals of discovery. - James J.", + "๐Ÿง  Itโ€™s extremely difficult to be simultaneously concerned with the end-user experience of" + " whatever it is that youโ€™re building and the architecture of the program that delivers that" + "experience. - James H.", + "๐Ÿง  Good design is a lot like clear thinking made visual. - Edward T.", + "๐Ÿš€ Innovation leads one to see the new in the old and distinguishes the ingenious from the" + " ingenuous. - Paul R.", + "๐Ÿ”ฎ The best way to predict the future is to invent it. - Alan K.", + "๐ŸŒŸ Perfection is achieved, not when there is nothing more to add, but when there is nothing" + " left to take away. - Antoine d.", + "๐Ÿ“ You canโ€™t improve what you donโ€™t measure. - Tom D.", +] + + +def wrap_quote(quote, width=80): + """ + Wraps the quote text to the given width. + """ + return textwrap.fill(quote, width=width) + + +def box_quote(quote): + """ + Returns a quote wrapped in a box with borders. + """ + # Wrap the quote first + wrapped_quote = wrap_quote(quote) + + # Calculate the width of the box + box_width = max(len(line) for line in wrapped_quote.split("\n")) + 2 # +2 for side borders + + # Create top and bottom border + top_bottom_border = "+" + "-" * (box_width) + "+" + + # Create the sides of the box + lines = wrapped_quote.split("\n") + boxed_lines = [f"{line.ljust(box_width - 2)}" for line in lines] + + # Return the full boxed quote + quote = "\n".join([top_bottom_border] + boxed_lines + [top_bottom_border]) + return f"\n {quote} \n" + + +def get_quote(): + """ + Returns a random inspirational quote related to system design formatted in a box. + """ + return box_quote(random.choice(make_something_great)) diff --git a/src/config/docs.py b/src/config/docs.py index d1896f3684..4f5d92e04c 100644 --- a/src/config/docs.py +++ b/src/config/docs.py @@ -18,3 +18,8 @@ class DocsConfig(BaseModel): GENERATE_UNTIL_FORK: str = "Osaka" """The fork until which documentation should be generated.""" + + DOCS_BASE_URL: str = "https://ethereum.github.io/execution-spec-tests" + + # Documentation URLs prefixed with `DOCS_URL__` to avoid conflicts with other URLs + DOCS_URL__WRITING_TESTS: str = f"{DOCS_BASE_URL}/main/writing_tests/" diff --git a/whitelist.txt b/whitelist.txt index 38e3d344b6..bfc77d2293 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -494,6 +494,7 @@ dedent dest exc extractall +fg fixturenames fspath funcargs