From 08bfa8dee56276f3d7ba113ccf631e30ef33c681 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 14:13:55 +0200 Subject: [PATCH 01/48] Create concept for namespaced custom components --- kpops/api/__init__.py | 3 +-- kpops/api/registry.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 826307308..a060b1618 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -250,8 +250,7 @@ def create_pipeline( environment: str | None, ) -> Pipeline: registry = Registry() - if kpops_config.components_module: - registry.find_components(kpops_config.components_module) + registry.load_components() registry.find_components("kpops.components") handlers = setup_handlers(kpops_config) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 2df483329..dd57e7335 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -3,11 +3,14 @@ import importlib import inspect import logging +import pkgutil import sys from dataclasses import dataclass, field from pathlib import Path +from types import ModuleType from typing import TYPE_CHECKING, TypeVar +import kpops.components from kpops import __name__ from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent @@ -38,6 +41,10 @@ def find_components(self, module_name: str) -> None: for _class in _find_classes(module_name, PipelineComponent): self._classes[_class.type] = _class + def load_components(self) -> None: + for _, module_name, _ in iter_namespace(kpops.components): + self.find_components(module_name) + def __getitem__(self, component_type: str) -> type[PipelineComponent]: try: return self._classes[component_type] @@ -74,3 +81,7 @@ def __filter_internal_kpops_classes(class_module: str, module_name: str) -> bool return class_module.startswith(KPOPS_MODULE) and not module_name.startswith( KPOPS_MODULE ) + + +def iter_namespace(ns_pkg: ModuleType) -> Iterator[pkgutil.ModuleInfo]: + return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") From a6a5b84b56eb69640faeaadbbcfdd20a85e9bf70 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 14:14:26 +0200 Subject: [PATCH 02/48] Remove components module from config --- docs/docs/resources/variables/config_env_vars.env | 3 --- docs/docs/resources/variables/config_env_vars.md | 1 - docs/docs/schema/config.json | 13 ------------- kpops/config.py | 4 ---- 4 files changed, 21 deletions(-) diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env index c4b4050e8..f558d4d19 100644 --- a/docs/docs/resources/variables/config_env_vars.env +++ b/docs/docs/resources/variables/config_env_vars.env @@ -4,9 +4,6 @@ # settings in `config.yaml`. Variables marked as required can instead # be set in the global config. # -# components_module -# Custom Python module defining project-specific KPOps components -KPOPS_COMPONENTS_MODULE # No default value, not required # pipeline_base_dir # Base directory to the pipelines (default is current working # directory) diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md index 171715ba5..8685acba0 100644 --- a/docs/docs/resources/variables/config_env_vars.md +++ b/docs/docs/resources/variables/config_env_vars.md @@ -2,7 +2,6 @@ These variables take precedence over the settings in `config.yaml`. Variables ma | Name | Default Value |Required| Description | Setting name | |--------------------------------------------------|----------------------------------------|--------|----------------------------------------------------------------------------------|-------------------------------------------| -|KPOPS_COMPONENTS_MODULE | |False |Custom Python module defining project-specific KPOps components |components_module | |KPOPS_PIPELINE_BASE_DIR |. |False |Base directory to the pipelines (default is current working directory) |pipeline_base_dir | |KPOPS_KAFKA_BROKERS | |True |The comma separated Kafka brokers address. |kafka_brokers | |KPOPS_TOPIC_NAME_CONFIG__DEFAULT_OUTPUT_TOPIC_NAME|${pipeline.name}-${component.name} |False |Configures the value for the variable ${output_topic_name} |topic_name_config.default_output_topic_name| diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json index 47aab93b1..949c42791 100644 --- a/docs/docs/schema/config.json +++ b/docs/docs/schema/config.json @@ -177,19 +177,6 @@ "additionalProperties": false, "description": "Global configuration for KPOps project.", "properties": { - "components_module": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Custom Python module defining project-specific KPOps components", - "title": "Components Module" - }, "create_namespace": { "default": false, "description": "Flag for `helm upgrade --install`. Create the release namespace if not present.", diff --git a/kpops/config.py b/kpops/config.py index 7efbbf0b7..5a9f25a3e 100644 --- a/kpops/config.py +++ b/kpops/config.py @@ -74,10 +74,6 @@ class KafkaConnectConfig(BaseSettings): class KpopsConfig(BaseSettings): """Global configuration for KPOps project.""" - components_module: str | None = Field( - default=None, - description="Custom Python module defining project-specific KPOps components", - ) pipeline_base_dir: Path = Field( default=Path(), description="Base directory to the pipelines (default is current working directory)", From 0e629af83302244916cdad2ad4e10ce37bb176a9 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:19:09 +0200 Subject: [PATCH 03/48] Temporarily disable gen-schema hook --- .github/workflows/ci.yaml | 4 ++-- .pre-commit-config.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 84577d1ab..fc45ec1f1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,8 +64,8 @@ jobs: poetry run pre-commit run pyright --all-files fi; - - name: Generate schema (kpops schema) - run: poetry run pre-commit run gen-schema --all-files --show-diff-on-failure + # - name: Generate schema (kpops schema) + # run: poetry run pre-commit run gen-schema --all-files --show-diff-on-failure - name: Generate CLI Usage docs (typer-cli) run: poetry run pre-commit run gen-docs-cli --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 10a660058..04e870704 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,14 +29,14 @@ repos: language: system types: [python] require_serial: true # run once for all files - - id: gen-schema - name: gen-schema - entry: python hooks/gen_schema.py - language: system - types: [python] - require_serial: true - pass_filenames: false - exclude: ^tests/.*snapshots/ + # - id: gen-schema + # name: gen-schema + # entry: python hooks/gen_schema.py + # language: system + # types: [python] + # require_serial: true + # pass_filenames: false + # exclude: ^tests/.*snapshots/ - id: gen-docs-cli name: gen-docs-cli entry: python hooks/gen_docs/gen_docs_cli_usage.py From b83c5a25bb458b9ce189e37821d5e42fd96e85e0 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:21:18 +0200 Subject: [PATCH 04/48] Temporarily disable schema command --- docs/docs/user/references/cli-commands.md | 34 ---------- kpops/cli/main.py | 80 +++++++++++------------ 2 files changed, 37 insertions(+), 77 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index 570563069..934e369c3 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -22,7 +22,6 @@ $ kpops [OPTIONS] COMMAND [ARGS]... * `init`: Initialize a new KPOps project. * `manifest`: Render final resource representation * `reset`: Reset pipeline steps -* `schema`: Generate JSON schema. ## `kpops clean` @@ -194,36 +193,3 @@ $ kpops reset [OPTIONS] PIPELINE_PATHS... * `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially. [default: no-parallel] * `--help`: Show this message and exit. - -## `kpops schema` - -Generate JSON schema. - -The schemas can be used to enable support for KPOps files in a text editor. - -**Usage**: - -```console -$ kpops schema [OPTIONS] SCOPE:{pipeline|defaults|config} -``` - -**Arguments**: - -* `SCOPE:{pipeline|defaults|config}`: - Scope of the generated schema - - - - - pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. - - - - - config: Schema of KpopsConfig. [required] - -**Options**: - -* `--config DIRECTORY`: Path to the dir containing config.yaml files [env var: KPOPS_CONFIG_PATH; default: .] -* `--include-stock-components / --no-include-stock-components`: Include the built-in KPOps components. [default: include-stock-components] -* `--help`: Show this message and exit. diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 395ab8e53..1562b43d2 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -7,15 +7,9 @@ import kpops from kpops import __version__ -from kpops.api.file_type import KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import collect_pipeline_paths -from kpops.config import ENV_PREFIX, KpopsConfig -from kpops.utils.gen_schema import ( - gen_config_schema, - gen_defaults_schema, - gen_pipeline_schema, -) +from kpops.config import ENV_PREFIX from kpops.utils.yaml import print_yaml app = typer.Typer(pretty_exceptions_enable=False) @@ -118,42 +112,42 @@ def init( kpops.init(path, config_include_opt=config_include_opt) -@app.command( - help=""" - Generate JSON schema. - - The schemas can be used to enable support for KPOps files in a text editor. - """ -) -def schema( - scope: KpopsFileType = typer.Argument( - ..., - show_default=False, - help=""" - Scope of the generated schema - \n\n\n - pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. - \n\n\n - config: Schema of KpopsConfig.""", - ), - config: Path = CONFIG_PATH_OPTION, - include_stock_components: bool = typer.Option( - default=True, help="Include the built-in KPOps components." - ), -) -> None: - match scope: - case KpopsFileType.PIPELINE: - kpops_config = KpopsConfig.create(config) - gen_pipeline_schema( - kpops_config.components_module, include_stock_components - ) - case KpopsFileType.DEFAULTS: - kpops_config = KpopsConfig.create(config) - gen_defaults_schema( - kpops_config.components_module, include_stock_components - ) - case KpopsFileType.CONFIG: - gen_config_schema() +# @app.command( +# help=""" +# Generate JSON schema. + +# The schemas can be used to enable support for KPOps files in a text editor. +# """ +# ) +# def schema( +# scope: KpopsFileType = typer.Argument( +# ..., +# show_default=False, +# help=""" +# Scope of the generated schema +# \n\n\n +# pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. +# \n\n\n +# config: Schema of KpopsConfig.""", +# ), +# config: Path = CONFIG_PATH_OPTION, +# include_stock_components: bool = typer.Option( +# default=True, help="Include the built-in KPOps components." +# ), +# ) -> None: +# match scope: +# case KpopsFileType.PIPELINE: +# kpops_config = KpopsConfig.create(config) +# gen_pipeline_schema( +# kpops_config.components_module, include_stock_components +# ) +# case KpopsFileType.DEFAULTS: +# kpops_config = KpopsConfig.create(config) +# gen_defaults_schema( +# kpops_config.components_module, include_stock_components +# ) +# case KpopsFileType.CONFIG: +# gen_config_schema() @app.command( From a4ff8e9c32a3d20182f0937f1ac7729684e07128 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:32:11 +0200 Subject: [PATCH 05/48] Temporarily disable schema test --- tests/cli/test_schema_generation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index 0741340e0..c10983286 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -76,6 +76,7 @@ class SubPipelineComponentCorrectDocstr(SubPipelineComponent): ) +@pytest.mark.skip() @pytest.mark.filterwarnings( "ignore:handlers", "ignore:config", "ignore:enrich", "ignore:validate" ) @@ -83,7 +84,7 @@ class TestGenSchema: @pytest.fixture def stock_components(self) -> list[type[PipelineComponent]]: registry = Registry() - registry.find_components("kpops.components") + registry.find_components() return list(registry._classes.values()) def test_gen_pipeline_schema_no_modules(self): From 406bfa3af4fc991e3aad92e939fcec13b911e0f6 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:31:30 +0200 Subject: [PATCH 06/48] Refactor Registry --- kpops/api/__init__.py | 3 +-- kpops/api/registry.py | 41 +++++++++++++++++++---------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index a060b1618..67697eb8f 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -250,8 +250,7 @@ def create_pipeline( environment: str | None, ) -> Pipeline: registry = Registry() - registry.load_components() - registry.find_components("kpops.components") + registry.find_components() handlers = setup_handlers(kpops_config) parser = PipelineGenerator(kpops_config, registry, handlers) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index dd57e7335..96370bf41 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -33,18 +33,14 @@ class Registry: _classes: ClassDict[PipelineComponent] = field(default_factory=dict, init=False) - def find_components(self, module_name: str) -> None: + def find_components(self) -> None: """Find all PipelineComponent subclasses in module. :param module_name: name of the python module. """ - for _class in _find_classes(module_name, PipelineComponent): + for _class in _find_classes(PipelineComponent): self._classes[_class.type] = _class - def load_components(self) -> None: - for _, module_name, _ in iter_namespace(kpops.components): - self.find_components(module_name) - def __getitem__(self, component_type: str) -> type[PipelineComponent]: try: return self._classes[component_type] @@ -53,27 +49,28 @@ def __getitem__(self, component_type: str) -> type[PipelineComponent]: raise ClassNotFoundError(msg) from ke -def find_class(module_name: str, baseclass: type[T]) -> type[T]: +def find_class(baseclass: type[T]) -> type[T]: try: - return next(_find_classes(module_name, baseclass)) + return next(_find_classes(baseclass)) except StopIteration as e: raise ClassNotFoundError from e -def _find_classes(module_name: str, baseclass: type[T]) -> Iterator[type[T]]: - module = importlib.import_module(module_name) - if module.__file__ and not module_name.startswith(KPOPS_MODULE): - file_path = Path(module.__file__) - try: - rel_path = file_path.relative_to(Path.cwd()) - log.debug(f"Picked up: {rel_path}") - except ValueError: - log.debug(f"Picked up: {file_path}") - for _, _class in inspect.getmembers(module, inspect.isclass): - if not __filter_internal_kpops_classes( - _class.__module__, module_name - ) and issubclass(_class, baseclass): - yield _class +def _find_classes(baseclass: type[T]) -> Iterator[type[T]]: + for _, module_name, _ in iter_namespace(kpops.components): + module = importlib.import_module(module_name) + if module.__file__ and not module_name.startswith(KPOPS_MODULE): + file_path = Path(module.__file__) + try: + rel_path = file_path.relative_to(Path.cwd()) + log.debug(f"Picked up: {rel_path}") + except ValueError: + log.debug(f"Picked up: {file_path}") + for _, _class in inspect.getmembers(module, inspect.isclass): + if not __filter_internal_kpops_classes( + _class.__module__, module_name + ) and issubclass(_class, baseclass): + yield _class def __filter_internal_kpops_classes(class_module: str, module_name: str) -> bool: From 61a242812328c6875d99bf58b9369c99f15d24e9 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:45:32 +0200 Subject: [PATCH 07/48] Temporarily disable test --- tests/api/test_registry.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 00daee08d..b8c629960 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -19,22 +19,18 @@ class Unrelated: pass -MODULE = SubComponent.__module__ - - +@pytest.mark.skip() def test_find_classes(): - gen = _find_classes(MODULE, PipelineComponent) + gen = _find_classes(PipelineComponent) assert next(gen) is SubComponent assert next(gen) is SubSubComponent with pytest.raises(StopIteration): next(gen) +@pytest.mark.skip() def test_find_builtin_classes(): - components = [ - class_.__name__ - for class_ in _find_classes("kpops.components", PipelineComponent) - ] + components = [class_.__name__ for class_ in _find_classes(PipelineComponent)] assert len(components) == 10 assert components == [ "HelmApp", @@ -50,18 +46,20 @@ def test_find_builtin_classes(): ] +@pytest.mark.skip() def test_find_class(): - assert find_class(MODULE, SubComponent) is SubComponent - assert find_class(MODULE, PipelineComponent) is SubComponent - assert find_class(MODULE, SchemaProvider) is CustomSchemaProvider + assert find_class(SubComponent) is SubComponent + assert find_class(PipelineComponent) is SubComponent + assert find_class(SchemaProvider) is CustomSchemaProvider with pytest.raises(ClassNotFoundError): - find_class(MODULE, dict) + find_class(dict) +@pytest.mark.skip() def test_registry(): registry = Registry() assert registry._classes == {} - registry.find_components(MODULE) + registry.find_components() assert registry._classes == { "sub-component": SubComponent, "sub-sub-component": SubSubComponent, From ef0733542645ba998f863602a96aa8da2f96d183 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:45:53 +0200 Subject: [PATCH 08/48] Refactor Schema Handler --- kpops/component_handlers/schema_handler/schema_handler.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/kpops/component_handlers/schema_handler/schema_handler.py b/kpops/component_handlers/schema_handler/schema_handler.py index 94b5cd3bf..d2b5b6ee7 100644 --- a/kpops/component_handlers/schema_handler/schema_handler.py +++ b/kpops/component_handlers/schema_handler/schema_handler.py @@ -29,18 +29,14 @@ def __init__(self, kpops_config: KpopsConfig) -> None: str(kpops_config.schema_registry.url), timeout=kpops_config.schema_registry.timeout, # pyright: ignore[reportArgumentType] ) - self.components_module = kpops_config.components_module @cached_property def schema_provider(self) -> SchemaProvider: try: - if not self.components_module: - msg = f"The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your {SchemaProvider.__name__} implementation exists." - raise ValueError(msg) - schema_provider_class = find_class(self.components_module, SchemaProvider) + schema_provider_class = find_class(SchemaProvider) return schema_provider_class() # pyright: ignore[reportAbstractUsage] except ClassNotFoundError as e: - msg = f"No schema provider found in components module {self.components_module}. Please implement the abstract method in {SchemaProvider.__module__}.{SchemaProvider.__name__}." + msg = f"No schema provider found. Please implement the abstract method in {SchemaProvider.__module__}.{SchemaProvider.__name__}." raise ValueError(msg) from e @classmethod From 1874705b91cc080c0852f15ba50f1a19ac2312f0 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:47:32 +0200 Subject: [PATCH 09/48] Update test --- .../schema_handler/test_schema_handler.py | 33 +------------------ 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/tests/component_handlers/schema_handler/test_schema_handler.py b/tests/component_handlers/schema_handler/test_schema_handler.py index 8d4052e54..de5249222 100644 --- a/tests/component_handlers/schema_handler/test_schema_handler.py +++ b/tests/component_handlers/schema_handler/test_schema_handler.py @@ -4,7 +4,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest -from pydantic import AnyHttpUrl, BaseModel, TypeAdapter +from pydantic import AnyHttpUrl, TypeAdapter from pytest_mock import MockerFixture from schema_registry.client.schema import AvroSchema from schema_registry.client.utils import SchemaVersion @@ -20,7 +20,6 @@ from kpops.utils.colorify import greenify, magentaify, yellowify from tests.pipeline.test_components import TestSchemaProvider -NON_EXISTING_PROVIDER_MODULE = BaseModel.__module__ TEST_SCHEMA_PROVIDER_MODULE = TestSchemaProvider.__module__ @@ -87,7 +86,6 @@ def kpops_config() -> KpopsConfig: enabled=True, url=TypeAdapter(AnyHttpUrl).validate_python("http://mock:8081"), # pyright: ignore[reportCallIssue,reportArgumentType] ), - components_module=TEST_SCHEMA_PROVIDER_MODULE, ) @@ -123,7 +121,6 @@ def test_should_lazy_load_schema_provider( def test_should_raise_value_error_if_schema_provider_class_not_found( kpops_config: KpopsConfig, ): - kpops_config.components_module = NON_EXISTING_PROVIDER_MODULE schema_handler = SchemaHandler(kpops_config) with pytest.raises( @@ -137,34 +134,6 @@ def test_should_raise_value_error_if_schema_provider_class_not_found( ) -@pytest.mark.parametrize( - ("components_module"), - [ - pytest.param( - None, - id="components_module = None", - ), - pytest.param( - "", - id="components_module = ''", - ), - ], -) -def test_should_raise_value_error_when_schema_provider_is_called_and_components_module_is_empty( - kpops_config: KpopsConfig, components_module: str | None -): - kpops_config.components_module = components_module - schema_handler = SchemaHandler.load_schema_handler(kpops_config) - assert schema_handler is not None - with pytest.raises( - ValueError, - match="The Schema Registry URL is set but you haven't specified the component module path. Please provide a valid component module path where your SchemaProvider implementation exists.", - ): - schema_handler.schema_provider.provide_schema( - "com.bakdata.kpops.test.SchemaHandlerTest", {} - ) - - @pytest.mark.asyncio() async def test_should_log_info_when_submit_schemas_that_not_exists_and_dry_run_true( to_section: ToSection, From 28adbbf9bb0c6e063d8966a7e3b8e3ab7b1053a8 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:47:40 +0200 Subject: [PATCH 10/48] Update test --- tests/api/test_handlers.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/tests/api/test_handlers.py b/tests/api/test_handlers.py index 18d35131b..e8bc71d9a 100644 --- a/tests/api/test_handlers.py +++ b/tests/api/test_handlers.py @@ -16,10 +16,7 @@ def test_set_up_handlers_with_no_schema_handler(mocker: MockerFixture): - config = KpopsConfig( - kafka_brokers="broker:9092", - components_module=MODULE, - ) + config = KpopsConfig(kafka_brokers="broker:9092") connector_handler_mock = mocker.patch(f"{HANDLER_MODULE}.KafkaConnectHandler") connector_handler = KafkaConnectHandler.from_kpops_config(config) connector_handler_mock.from_kpops_config.return_value = connector_handler From 62a490e0be535047d579c3435086cc49aa49d5ab Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 16:48:36 +0200 Subject: [PATCH 11/48] Update snapshot --- .../test_init/test_init_project/config_include_opt.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml index 5c251a22c..3c86a269a 100644 --- a/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml +++ b/tests/cli/snapshots/test_init/test_init_project/config_include_opt.yaml @@ -4,7 +4,6 @@ kafka_brokers: null # Non-required fields -components_module: null create_namespace: false helm_config: api_version: null From e7418e5585a1bc5f4bb5998ef0030ef778662f9c Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 18:03:44 +0200 Subject: [PATCH 12/48] Try testing --- tests/api/test_registry.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index b8c629960..a2b798a21 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -1,9 +1,14 @@ from __future__ import annotations +import importlib +import shutil +from pathlib import Path + import pytest +from pytest_mock import MockerFixture from kpops.api.exception import ClassNotFoundError -from kpops.api.registry import Registry, _find_classes, find_class +from kpops.api.registry import Registry, _find_classes, find_class, iter_namespace from kpops.component_handlers.schema_handler.schema_provider import SchemaProvider from kpops.components.base_components.pipeline_component import PipelineComponent from tests.cli.resources.custom_module import CustomSchemaProvider @@ -19,6 +24,26 @@ class Unrelated: pass +@pytest.fixture(autouse=True) +def custom_components(mocker: MockerFixture): + src = Path("tests/pipeline/test_components") + dst = Path("kpops/components/test_components") + try: + shutil.copytree(src, dst) + yield + finally: + shutil.rmtree(dst) + + +def test_iter_namespace(): + components_module = importlib.import_module("kpops.components") + assert [module_name for _, module_name, _ in iter_namespace(components_module)] == [ + "base_components", + "streams_bootstrap", + "test_components", + ] + + @pytest.mark.skip() def test_find_classes(): gen = _find_classes(PipelineComponent) From 224a8715d5f2489f19c6a83e3060922b87f18dfa Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 19:11:25 +0200 Subject: [PATCH 13/48] Refactor registry --- kpops/api/registry.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 96370bf41..aa689286a 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -38,7 +38,8 @@ def find_components(self) -> None: :param module_name: name of the python module. """ - for _class in _find_classes(PipelineComponent): + custom_modules = self.iter_custom_modules() + for _class in _find_classes(*custom_modules, base=PipelineComponent): self._classes[_class.type] = _class def __getitem__(self, component_type: str) -> type[PipelineComponent]: @@ -48,28 +49,37 @@ def __getitem__(self, component_type: str) -> type[PipelineComponent]: msg = f"Could not find a component of type {component_type}" raise ClassNotFoundError(msg) from ke + @staticmethod + def iter_custom_modules() -> Iterator[ModuleType]: + for _, module_name, _ in _iter_namespace(kpops.components): + yield import_module(module_name) -def find_class(baseclass: type[T]) -> type[T]: + +def find_class(*modules: ModuleType, base: type[T]) -> type[T]: try: - return next(_find_classes(baseclass)) + return next(_find_classes(*modules, base=base)) except StopIteration as e: raise ClassNotFoundError from e -def _find_classes(baseclass: type[T]) -> Iterator[type[T]]: - for _, module_name, _ in iter_namespace(kpops.components): - module = importlib.import_module(module_name) - if module.__file__ and not module_name.startswith(KPOPS_MODULE): - file_path = Path(module.__file__) - try: - rel_path = file_path.relative_to(Path.cwd()) - log.debug(f"Picked up: {rel_path}") - except ValueError: - log.debug(f"Picked up: {file_path}") +def import_module(module_name: str) -> ModuleType: + module = importlib.import_module(module_name) + if module.__file__ and not module_name.startswith(KPOPS_MODULE): + file_path = Path(module.__file__) + try: + rel_path = file_path.relative_to(Path.cwd()) + log.debug(f"Picked up: {rel_path}") + except ValueError: + log.debug(f"Picked up: {file_path}") + return module + + +def _find_classes(*modules: ModuleType, base: type[T]) -> Iterator[type[T]]: + for module in modules: for _, _class in inspect.getmembers(module, inspect.isclass): if not __filter_internal_kpops_classes( - _class.__module__, module_name - ) and issubclass(_class, baseclass): + _class.__module__, module.__name__ + ) and issubclass(_class, base): yield _class @@ -80,5 +90,5 @@ def __filter_internal_kpops_classes(class_module: str, module_name: str) -> bool ) -def iter_namespace(ns_pkg: ModuleType) -> Iterator[pkgutil.ModuleInfo]: +def _iter_namespace(ns_pkg: ModuleType) -> Iterator[pkgutil.ModuleInfo]: return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") From 43d06ff3c88f32e3632d5d177a45a55f0c41332a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 19:15:12 +0200 Subject: [PATCH 14/48] Update Schema Handler --- kpops/api/registry.py | 7 ++++--- kpops/component_handlers/schema_handler/schema_handler.py | 6 ++++-- tests/api/test_registry.py | 6 ++++-- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index aa689286a..dbddc936b 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -10,7 +10,6 @@ from types import ModuleType from typing import TYPE_CHECKING, TypeVar -import kpops.components from kpops import __name__ from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent @@ -38,7 +37,7 @@ def find_components(self) -> None: :param module_name: name of the python module. """ - custom_modules = self.iter_custom_modules() + custom_modules = self.iter_component_modules() for _class in _find_classes(*custom_modules, base=PipelineComponent): self._classes[_class.type] = _class @@ -50,7 +49,9 @@ def __getitem__(self, component_type: str) -> type[PipelineComponent]: raise ClassNotFoundError(msg) from ke @staticmethod - def iter_custom_modules() -> Iterator[ModuleType]: + def iter_component_modules() -> Iterator[ModuleType]: + import kpops.components + for _, module_name, _ in _iter_namespace(kpops.components): yield import_module(module_name) diff --git a/kpops/component_handlers/schema_handler/schema_handler.py b/kpops/component_handlers/schema_handler/schema_handler.py index d2b5b6ee7..ea4b01bb3 100644 --- a/kpops/component_handlers/schema_handler/schema_handler.py +++ b/kpops/component_handlers/schema_handler/schema_handler.py @@ -9,7 +9,7 @@ from schema_registry.client.schema import AvroSchema from kpops.api.exception import ClassNotFoundError -from kpops.api.registry import find_class +from kpops.api.registry import Registry, find_class from kpops.component_handlers.schema_handler.schema_provider import ( Schema, SchemaProvider, @@ -33,7 +33,9 @@ def __init__(self, kpops_config: KpopsConfig) -> None: @cached_property def schema_provider(self) -> SchemaProvider: try: - schema_provider_class = find_class(SchemaProvider) + schema_provider_class = find_class( + *Registry.iter_component_modules(), base=SchemaProvider + ) return schema_provider_class() # pyright: ignore[reportAbstractUsage] except ClassNotFoundError as e: msg = f"No schema provider found. Please implement the abstract method in {SchemaProvider.__module__}.{SchemaProvider.__name__}." diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index a2b798a21..7502e5fd4 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -8,7 +8,7 @@ from pytest_mock import MockerFixture from kpops.api.exception import ClassNotFoundError -from kpops.api.registry import Registry, _find_classes, find_class, iter_namespace +from kpops.api.registry import Registry, _find_classes, _iter_namespace, find_class from kpops.component_handlers.schema_handler.schema_provider import SchemaProvider from kpops.components.base_components.pipeline_component import PipelineComponent from tests.cli.resources.custom_module import CustomSchemaProvider @@ -37,7 +37,9 @@ def custom_components(mocker: MockerFixture): def test_iter_namespace(): components_module = importlib.import_module("kpops.components") - assert [module_name for _, module_name, _ in iter_namespace(components_module)] == [ + assert [ + module_name for _, module_name, _ in _iter_namespace(components_module) + ] == [ "base_components", "streams_bootstrap", "test_components", From 628ca21ca3bff31c447ddd4a82b3a8fceccef174 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 19:17:28 +0200 Subject: [PATCH 15/48] Mark todo --- kpops/api/registry.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index dbddc936b..fc0623329 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -65,6 +65,7 @@ def find_class(*modules: ModuleType, base: type[T]) -> type[T]: def import_module(module_name: str) -> ModuleType: module = importlib.import_module(module_name) + # TODO: remove? unnecessary now if module.__file__ and not module_name.startswith(KPOPS_MODULE): file_path = Path(module.__file__) try: From c40ae5c4af0c2b5b481d6eb2bc36832a2cfb9b50 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 19:31:18 +0200 Subject: [PATCH 16/48] Update tests --- kpops/api/registry.py | 9 +++-- tests/api/test_registry.py | 75 ++++++++++++++++++++++++++++---------- 2 files changed, 61 insertions(+), 23 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index fc0623329..81b706dae 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -5,6 +5,7 @@ import logging import pkgutil import sys +from collections.abc import Iterable from dataclasses import dataclass, field from pathlib import Path from types import ModuleType @@ -38,7 +39,7 @@ def find_components(self) -> None: :param module_name: name of the python module. """ custom_modules = self.iter_component_modules() - for _class in _find_classes(*custom_modules, base=PipelineComponent): + for _class in _find_classes(custom_modules, base=PipelineComponent): self._classes[_class.type] = _class def __getitem__(self, component_type: str) -> type[PipelineComponent]: @@ -56,9 +57,9 @@ def iter_component_modules() -> Iterator[ModuleType]: yield import_module(module_name) -def find_class(*modules: ModuleType, base: type[T]) -> type[T]: +def find_class(modules: Iterable[ModuleType], base: type[T]) -> type[T]: try: - return next(_find_classes(*modules, base=base)) + return next(_find_classes(modules, base=base)) except StopIteration as e: raise ClassNotFoundError from e @@ -76,7 +77,7 @@ def import_module(module_name: str) -> ModuleType: return module -def _find_classes(*modules: ModuleType, base: type[T]) -> Iterator[type[T]]: +def _find_classes(modules: Iterable[ModuleType], base: type[T]) -> Iterator[type[T]]: for module in modules: for _, _class in inspect.getmembers(module, inspect.isclass): if not __filter_internal_kpops_classes( diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 7502e5fd4..ae1203d3f 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -3,6 +3,7 @@ import importlib import shutil from pathlib import Path +from types import ModuleType import pytest from pytest_mock import MockerFixture @@ -10,7 +11,18 @@ from kpops.api.exception import ClassNotFoundError from kpops.api.registry import Registry, _find_classes, _iter_namespace, find_class from kpops.component_handlers.schema_handler.schema_provider import SchemaProvider -from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components import ( + HelmApp, + KafkaApp, + KafkaConnector, + KafkaSinkConnector, + KafkaSourceConnector, + KubernetesApp, + PipelineComponent, + ProducerApp, + StreamsApp, + StreamsBootstrap, +) from tests.cli.resources.custom_module import CustomSchemaProvider @@ -24,7 +36,10 @@ class Unrelated: pass -@pytest.fixture(autouse=True) +MODULE = SubComponent.__module__ + + +@pytest.fixture() def custom_components(mocker: MockerFixture): src = Path("tests/pipeline/test_components") dst = Path("kpops/components/test_components") @@ -35,29 +50,45 @@ def custom_components(mocker: MockerFixture): shutil.rmtree(dst) +@pytest.mark.usefixtures("custom_components") def test_iter_namespace(): components_module = importlib.import_module("kpops.components") assert [ module_name for _, module_name, _ in _iter_namespace(components_module) ] == [ - "base_components", - "streams_bootstrap", - "test_components", + "kpops.components.base_components", + "kpops.components.streams_bootstrap", + "kpops.components.test_components", + ] + + +@pytest.mark.usefixtures("custom_components") +def test_iter_component_modules(): + assert [module.__name__ for module in Registry.iter_component_modules()] == [ + "kpops.components.base_components", + "kpops.components.streams_bootstrap", + "kpops.components.test_components", ] -@pytest.mark.skip() -def test_find_classes(): - gen = _find_classes(PipelineComponent) +@pytest.fixture() +def module() -> ModuleType: + return importlib.import_module(MODULE) + + +def test_find_classes(module: ModuleType): + gen = _find_classes([module], PipelineComponent) assert next(gen) is SubComponent assert next(gen) is SubSubComponent with pytest.raises(StopIteration): next(gen) -@pytest.mark.skip() def test_find_builtin_classes(): - components = [class_.__name__ for class_ in _find_classes(PipelineComponent)] + modules = Registry.iter_component_modules() + components = [ + class_.__name__ for class_ in _find_classes(modules, base=PipelineComponent) + ] assert len(components) == 10 assert components == [ "HelmApp", @@ -73,23 +104,29 @@ def test_find_builtin_classes(): ] -@pytest.mark.skip() -def test_find_class(): - assert find_class(SubComponent) is SubComponent - assert find_class(PipelineComponent) is SubComponent - assert find_class(SchemaProvider) is CustomSchemaProvider +def test_find_class(module: ModuleType): + assert find_class([module], base=SubComponent) is SubComponent + assert find_class([module], base=PipelineComponent) is SubComponent + assert find_class([module], base=SchemaProvider) is CustomSchemaProvider with pytest.raises(ClassNotFoundError): - find_class(dict) + find_class([module], base=dict) -@pytest.mark.skip() def test_registry(): registry = Registry() assert registry._classes == {} registry.find_components() assert registry._classes == { - "sub-component": SubComponent, - "sub-sub-component": SubSubComponent, + "helm-app": HelmApp, + "kafka-app": KafkaApp, + "kafka-connector": KafkaConnector, + "kafka-sink-connector": KafkaSinkConnector, + "kafka-source-connector": KafkaSourceConnector, + "kubernetes-app": KubernetesApp, + "pipeline-component": PipelineComponent, + "producer-app": ProducerApp, + "streams-app": StreamsApp, + "streams-bootstrap": StreamsBootstrap, } assert registry["sub-component"] is SubComponent assert registry["sub-sub-component"] is SubSubComponent From 246541ff3cbe6fce739a0620b1ddca94126ccef2 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 19:49:57 +0200 Subject: [PATCH 17/48] Improve --- kpops/api/registry.py | 2 +- tests/api/test_registry.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 81b706dae..7e67a4e1a 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -87,7 +87,7 @@ def _find_classes(modules: Iterable[ModuleType], base: type[T]) -> Iterator[type def __filter_internal_kpops_classes(class_module: str, module_name: str) -> bool: - # filter out internal kpops classes and components unless specifically requested + """Filter out internal kpops classes and components unless specifically requested.""" return class_module.startswith(KPOPS_MODULE) and not module_name.startswith( KPOPS_MODULE ) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index ae1203d3f..8d26b04ba 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -128,7 +128,7 @@ def test_registry(): "streams-app": StreamsApp, "streams-bootstrap": StreamsBootstrap, } - assert registry["sub-component"] is SubComponent - assert registry["sub-sub-component"] is SubSubComponent + for _type, _class in registry._classes.items(): + assert registry[_type] is _class with pytest.raises(ClassNotFoundError): registry["doesnt-exist"] From f76c90df0f8a78651adc0a56e127014116af37bc Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:04:18 +0200 Subject: [PATCH 18/48] Fix include package itself for module discovery --- kpops/api/registry.py | 11 +++++++---- tests/api/test_registry.py | 25 ++----------------------- 2 files changed, 9 insertions(+), 27 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 7e67a4e1a..55c47f029 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -53,8 +53,8 @@ def __getitem__(self, component_type: str) -> type[PipelineComponent]: def iter_component_modules() -> Iterator[ModuleType]: import kpops.components - for _, module_name, _ in _iter_namespace(kpops.components): - yield import_module(module_name) + yield kpops.components + yield from _iter_namespace(kpops.components) def find_class(modules: Iterable[ModuleType], base: type[T]) -> type[T]: @@ -93,5 +93,8 @@ def __filter_internal_kpops_classes(class_module: str, module_name: str) -> bool ) -def _iter_namespace(ns_pkg: ModuleType) -> Iterator[pkgutil.ModuleInfo]: - return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".") +def _iter_namespace(ns_pkg: ModuleType) -> Iterator[ModuleType]: + for _, module_name, _ in pkgutil.iter_modules( + ns_pkg.__path__, ns_pkg.__name__ + "." + ): + yield import_module(module_name) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 8d26b04ba..e38032904 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -53,9 +53,7 @@ def custom_components(mocker: MockerFixture): @pytest.mark.usefixtures("custom_components") def test_iter_namespace(): components_module = importlib.import_module("kpops.components") - assert [ - module_name for _, module_name, _ in _iter_namespace(components_module) - ] == [ + assert [module.__name__ for module in _iter_namespace(components_module)] == [ "kpops.components.base_components", "kpops.components.streams_bootstrap", "kpops.components.test_components", @@ -65,6 +63,7 @@ def test_iter_namespace(): @pytest.mark.usefixtures("custom_components") def test_iter_component_modules(): assert [module.__name__ for module in Registry.iter_component_modules()] == [ + "kpops.components", "kpops.components.base_components", "kpops.components.streams_bootstrap", "kpops.components.test_components", @@ -84,26 +83,6 @@ def test_find_classes(module: ModuleType): next(gen) -def test_find_builtin_classes(): - modules = Registry.iter_component_modules() - components = [ - class_.__name__ for class_ in _find_classes(modules, base=PipelineComponent) - ] - assert len(components) == 10 - assert components == [ - "HelmApp", - "KafkaApp", - "KafkaConnector", - "KafkaSinkConnector", - "KafkaSourceConnector", - "KubernetesApp", - "PipelineComponent", - "ProducerApp", - "StreamsApp", - "StreamsBootstrap", - ] - - def test_find_class(module: ModuleType): assert find_class([module], base=SubComponent) is SubComponent assert find_class([module], base=PipelineComponent) is SubComponent From 9ecbaa646a76774f74be927ad97c67c456ae41ee Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:09:16 +0200 Subject: [PATCH 19/48] Fix Schema Handler --- kpops/component_handlers/schema_handler/schema_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kpops/component_handlers/schema_handler/schema_handler.py b/kpops/component_handlers/schema_handler/schema_handler.py index ea4b01bb3..c23bea4a3 100644 --- a/kpops/component_handlers/schema_handler/schema_handler.py +++ b/kpops/component_handlers/schema_handler/schema_handler.py @@ -34,7 +34,7 @@ def __init__(self, kpops_config: KpopsConfig) -> None: def schema_provider(self) -> SchemaProvider: try: schema_provider_class = find_class( - *Registry.iter_component_modules(), base=SchemaProvider + Registry.iter_component_modules(), base=SchemaProvider ) return schema_provider_class() # pyright: ignore[reportAbstractUsage] except ClassNotFoundError as e: From d0a37c9406aadbbe772fc81d6ed8c3b71d0be237 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:27:18 +0200 Subject: [PATCH 20/48] Update Schema Handler tests --- .../schema_handler/test_schema_handler.py | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/tests/component_handlers/schema_handler/test_schema_handler.py b/tests/component_handlers/schema_handler/test_schema_handler.py index de5249222..671b9915b 100644 --- a/tests/component_handlers/schema_handler/test_schema_handler.py +++ b/tests/component_handlers/schema_handler/test_schema_handler.py @@ -1,5 +1,7 @@ import json import logging +import shutil +from pathlib import Path from unittest import mock from unittest.mock import AsyncMock, MagicMock @@ -20,9 +22,6 @@ from kpops.utils.colorify import greenify, magentaify, yellowify from tests.pipeline.test_components import TestSchemaProvider -TEST_SCHEMA_PROVIDER_MODULE = TestSchemaProvider.__module__ - - log = logging.getLogger("SchemaHandler") @@ -47,13 +46,6 @@ def log_warning_mock(mocker: MockerFixture) -> MagicMock: ) -@pytest.fixture(autouse=False) -def find_class_mock(mocker: MockerFixture) -> MagicMock: - return mocker.patch( - "kpops.component_handlers.schema_handler.schema_handler.find_class" - ) - - @pytest.fixture(autouse=True) def schema_registry_mock(mocker: MockerFixture) -> AsyncMock: schema_registry_mock_constructor = mocker.patch( @@ -89,11 +81,19 @@ def kpops_config() -> KpopsConfig: ) +@pytest.fixture() +def custom_components(mocker: MockerFixture): + src = Path("tests/pipeline/test_components") + dst = Path("kpops/components/test_components") + try: + shutil.copytree(src, dst) + yield + finally: + shutil.rmtree(dst) + + def test_load_schema_handler(kpops_config: KpopsConfig): - assert isinstance( - SchemaHandler.load_schema_handler(kpops_config), - SchemaHandler, - ) + assert isinstance(SchemaHandler.load_schema_handler(kpops_config), SchemaHandler) config_disable = kpops_config.model_copy() config_disable.schema_registry = SchemaRegistryConfig(enabled=False) @@ -101,9 +101,8 @@ def test_load_schema_handler(kpops_config: KpopsConfig): assert SchemaHandler.load_schema_handler(config_disable) is None -def test_should_lazy_load_schema_provider( - find_class_mock: MagicMock, kpops_config: KpopsConfig -): +@pytest.mark.usefixtures("custom_components") +def test_should_lazy_load_schema_provider(kpops_config: KpopsConfig): schema_handler = SchemaHandler.load_schema_handler(kpops_config) assert schema_handler is not None @@ -115,7 +114,7 @@ def test_should_lazy_load_schema_provider( "com.bakdata.kpops.test.SomeOtherSchemaClass", {} ) - find_class_mock.assert_called_once_with(TEST_SCHEMA_PROVIDER_MODULE, SchemaProvider) + assert isinstance(schema_handler.schema_provider, TestSchemaProvider) def test_should_raise_value_error_if_schema_provider_class_not_found( @@ -125,7 +124,7 @@ def test_should_raise_value_error_if_schema_provider_class_not_found( with pytest.raises( ValueError, - match="No schema provider found in components module pydantic.main. " + match="No schema provider found. " "Please implement the abstract method in " f"{SchemaProvider.__module__}.{SchemaProvider.__name__}.", ): @@ -135,6 +134,7 @@ def test_should_raise_value_error_if_schema_provider_class_not_found( @pytest.mark.asyncio() +@pytest.mark.usefixtures("custom_components") async def test_should_log_info_when_submit_schemas_that_not_exists_and_dry_run_true( to_section: ToSection, log_info_mock: MagicMock, @@ -154,6 +154,7 @@ async def test_should_log_info_when_submit_schemas_that_not_exists_and_dry_run_t @pytest.mark.asyncio() +@pytest.mark.usefixtures("custom_components") async def test_should_log_info_when_submit_schemas_that_exists_and_dry_run_true( topic_config: TopicConfig, to_section: ToSection, @@ -176,6 +177,7 @@ async def test_should_log_info_when_submit_schemas_that_exists_and_dry_run_true( @pytest.mark.asyncio() +@pytest.mark.usefixtures("custom_components") async def test_should_raise_exception_when_submit_schema_that_exists_and_not_compatible_and_dry_run_true( topic_config: TopicConfig, to_section: ToSection, @@ -213,6 +215,7 @@ async def test_should_raise_exception_when_submit_schema_that_exists_and_not_com @pytest.mark.asyncio() +@pytest.mark.usefixtures("custom_components") async def test_should_log_debug_when_submit_schema_that_exists_and_registered_under_version_and_dry_run_true( topic_config: TopicConfig, to_section: ToSection, @@ -248,6 +251,7 @@ async def test_should_log_debug_when_submit_schema_that_exists_and_registered_un @pytest.mark.asyncio() +@pytest.mark.usefixtures("custom_components") async def test_should_submit_non_existing_schema_when_not_dry( topic_config: TopicConfig, to_section: ToSection, From 534b95f3494bf890535e294af1d1a4b8688ee55f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:31:37 +0200 Subject: [PATCH 21/48] Move common fixture to conftest --- tests/api/test_registry.py | 14 -------------- .../schema_handler/test_schema_handler.py | 13 ------------- tests/conftest.py | 13 +++++++++++++ 3 files changed, 13 insertions(+), 27 deletions(-) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index e38032904..4b4d4f37c 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -1,12 +1,9 @@ from __future__ import annotations import importlib -import shutil -from pathlib import Path from types import ModuleType import pytest -from pytest_mock import MockerFixture from kpops.api.exception import ClassNotFoundError from kpops.api.registry import Registry, _find_classes, _iter_namespace, find_class @@ -39,17 +36,6 @@ class Unrelated: MODULE = SubComponent.__module__ -@pytest.fixture() -def custom_components(mocker: MockerFixture): - src = Path("tests/pipeline/test_components") - dst = Path("kpops/components/test_components") - try: - shutil.copytree(src, dst) - yield - finally: - shutil.rmtree(dst) - - @pytest.mark.usefixtures("custom_components") def test_iter_namespace(): components_module = importlib.import_module("kpops.components") diff --git a/tests/component_handlers/schema_handler/test_schema_handler.py b/tests/component_handlers/schema_handler/test_schema_handler.py index 671b9915b..716f2f482 100644 --- a/tests/component_handlers/schema_handler/test_schema_handler.py +++ b/tests/component_handlers/schema_handler/test_schema_handler.py @@ -1,7 +1,5 @@ import json import logging -import shutil -from pathlib import Path from unittest import mock from unittest.mock import AsyncMock, MagicMock @@ -81,17 +79,6 @@ def kpops_config() -> KpopsConfig: ) -@pytest.fixture() -def custom_components(mocker: MockerFixture): - src = Path("tests/pipeline/test_components") - dst = Path("kpops/components/test_components") - try: - shutil.copytree(src, dst) - yield - finally: - shutil.rmtree(dst) - - def test_load_schema_handler(kpops_config: KpopsConfig): assert isinstance(SchemaHandler.load_schema_handler(kpops_config), SchemaHandler) diff --git a/tests/conftest.py b/tests/conftest.py index 5fb77a415..0e23be87c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,8 @@ import logging import os +import shutil from collections.abc import Iterator +from pathlib import Path from unittest import mock import pytest @@ -38,3 +40,14 @@ def mock_env() -> Iterator[Environment]: def load_yaml_file_clear_cache() -> Iterator[None]: yield load_yaml_file.cache.clear() # pyright: ignore[reportFunctionMemberAccess] + + +@pytest.fixture() +def custom_components(): + src = Path("tests/pipeline/test_components") + dst = Path("kpops/components/test_components") + try: + shutil.copytree(src, dst) + yield + finally: + shutil.rmtree(dst) From fb6f5786fb220b805d3d68fa597ef9ed0a98e66e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:33:09 +0200 Subject: [PATCH 22/48] Fix test generate --- tests/pipeline/test_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 915ae17d7..a209071e3 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -22,7 +22,7 @@ RESOURCE_PATH = Path(__file__).parent / "resources" -@pytest.mark.usefixtures("mock_env", "load_yaml_file_clear_cache") +@pytest.mark.usefixtures("mock_env", "load_yaml_file_clear_cache", "custom_components") class TestGenerate: @pytest.fixture(autouse=True) def log_info(self, mocker: MockerFixture) -> MagicMock: From 0ff01737ca009db8f6d7aecb40ccbac9eab82c89 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:38:54 +0200 Subject: [PATCH 23/48] Rename method --- kpops/api/__init__.py | 2 +- kpops/api/registry.py | 6 +++--- tests/api/test_registry.py | 2 +- tests/cli/test_schema_generation.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index 67697eb8f..aa7133fa2 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -250,7 +250,7 @@ def create_pipeline( environment: str | None, ) -> Pipeline: registry = Registry() - registry.find_components() + registry.discover_components() handlers = setup_handlers(kpops_config) parser = PipelineGenerator(kpops_config, registry, handlers) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 55c47f029..592f481d2 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -33,10 +33,10 @@ class Registry: _classes: ClassDict[PipelineComponent] = field(default_factory=dict, init=False) - def find_components(self) -> None: - """Find all PipelineComponent subclasses in module. + def discover_components(self) -> None: + """Discover first- and third-party KPOps components. - :param module_name: name of the python module. + That is all classes inheriting from PipelineComponent. """ custom_modules = self.iter_component_modules() for _class in _find_classes(custom_modules, base=PipelineComponent): diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 4b4d4f37c..46e62f17f 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -80,7 +80,7 @@ def test_find_class(module: ModuleType): def test_registry(): registry = Registry() assert registry._classes == {} - registry.find_components() + registry.discover_components() assert registry._classes == { "helm-app": HelmApp, "kafka-app": KafkaApp, diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index c10983286..a6fe57380 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -84,7 +84,7 @@ class TestGenSchema: @pytest.fixture def stock_components(self) -> list[type[PipelineComponent]]: registry = Registry() - registry.find_components() + registry.discover_components() return list(registry._classes.values()) def test_gen_pipeline_schema_no_modules(self): From 7cfd06ad53af37b4728841b19b39bd5150873367 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:41:48 +0200 Subject: [PATCH 24/48] Update gen docs hook --- hooks/gen_docs/gen_docs_components.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index 10bb40af7..b6c733267 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -8,12 +8,15 @@ import yaml from hooks import ROOT -from kpops.api.registry import _find_classes +from kpops.api.registry import Registry from kpops.components import KafkaConnector, PipelineComponent from kpops.utils.colorify import redify, yellowify from kpops.utils.pydantic import issubclass_patched from kpops.utils.yaml import load_yaml_file +registry = Registry() +registry.discover_components() + PATH_KPOPS_MAIN = ROOT / "kpops/cli/main.py" PATH_CLI_COMMANDS_DOC = ROOT / "docs/docs/user/references/cli-commands.md" PATH_DOCS_RESOURCES = ROOT / "docs/docs/resources" @@ -33,7 +36,7 @@ (PATH_DOCS_RESOURCES / "pipeline-defaults/headers").iterdir(), ) -KPOPS_COMPONENTS = tuple(_find_classes("kpops.components", PipelineComponent)) +KPOPS_COMPONENTS = tuple(registry._classes.values()) KPOPS_COMPONENTS_SECTIONS = { component.type: [ field_name From 0b9404699ead5aa20277110de4e5fbd16fdee91d Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:53:05 +0200 Subject: [PATCH 25/48] Fix gen schema hook --- kpops/utils/gen_schema.py | 76 ++++++++------------------------------- 1 file changed, 14 insertions(+), 62 deletions(-) diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index 3b4ce5ad3..6cf326ed0 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -20,7 +20,7 @@ ModelFieldsSchema, ) -from kpops.api.registry import _find_classes +from kpops.api.registry import Registry from kpops.components import ( PipelineComponent, ) @@ -32,6 +32,10 @@ class MultiComponentGenerateJsonSchema(GenerateJsonSchema): ... log = logging.getLogger("") +registry = Registry() +registry.discover_components() +COMPONENTS = tuple(registry._classes.values()) + def print_schema(model: type[BaseModel]) -> None: schema = model_json_schema(model, by_alias=True) @@ -39,7 +43,6 @@ def print_schema(model: type[BaseModel]) -> None: def _is_valid_component( - defined_component_types: set[str], component: type[PipelineComponent], allow_abstract: bool, ) -> bool: @@ -54,65 +57,10 @@ def _is_valid_component( ): log.warning(f"SKIPPED {component.__name__}, component is abstract.") return False - if component.type in defined_component_types: - log.warning(f"SKIPPED {component.__name__}, component type must be unique.") - return False - defined_component_types.add(component.type) return True -def _add_components( - components_module: str, - allow_abstract: bool, - components: tuple[type[PipelineComponent], ...] | None = None, -) -> tuple[type[PipelineComponent], ...]: - """Add components to a components tuple. - - If an empty tuple is provided or it is not provided at all, the components - types from the given module are 'tupled' - - :param components_module: Python module. Only the classes that inherit from - PipelineComponent will be considered. - :param components: Tuple of components to which to add, defaults to () - :return: Extended tuple - """ - if components is None: - components = () - # Set of existing types, against which to check the new ones - defined_component_types = {component.type for component in components} - custom_components = ( - component - for component in _find_classes(components_module, PipelineComponent) - if _is_valid_component(defined_component_types, component, allow_abstract) - ) - components += tuple(custom_components) - return components - - -def find_components( - components_module: str | None, - include_stock_components: bool, - include_abstract: bool = False, -) -> tuple[type[PipelineComponent], ...]: - if not (include_stock_components or components_module): - msg = "No components are provided, no schema is generated." - raise RuntimeError(msg) - # Add stock components if enabled - components: tuple[type[PipelineComponent], ...] = () - if include_stock_components: - components = _add_components("kpops.components", include_abstract) - # Add custom components if provided - if components_module: - components = _add_components(components_module, include_abstract, components) - if not components: - msg = "No valid components found." - raise RuntimeError(msg) - return components - - -def gen_pipeline_schema( - components_module: str | None = None, include_stock_components: bool = True -) -> None: +def gen_pipeline_schema() -> None: """Generate a json schema from the models of pipeline components. :param components_module: Python module. Only the classes that inherit from @@ -120,8 +68,9 @@ def gen_pipeline_schema( :param include_stock_components: Whether to include the stock components, defaults to True """ - components = find_components(components_module, include_stock_components) - + components = [ + component for component in COMPONENTS if _is_valid_component(component, False) + ] # re-assign component type as Literal to work as discriminator for component in components: component.model_fields["type"] = FieldInfo( @@ -141,7 +90,7 @@ def gen_pipeline_schema( ), ) - PipelineComponents = Union[components] # type: ignore[valid-type] + PipelineComponents = Union[tuple(components)] # type: ignore[valid-type] AnnotatedPipelineComponents = Annotated[ PipelineComponents, Field(discriminator="type") ] @@ -157,7 +106,10 @@ class PipelineSchema(RootModel): def gen_defaults_schema( components_module: str | None = None, include_stock_components: bool = True ) -> None: - components = find_components(components_module, include_stock_components, True) + # components = find_components(components_module, include_stock_components, True) + components = [ + component for component in COMPONENTS if _is_valid_component(component, True) + ] components_mapping: dict[str, Any] = { component.type: (component, ...) for component in components } From eab2d4f5113854211d9e78f7fe660300676b8dbd Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:53:38 +0200 Subject: [PATCH 26/48] Revert "Temporarily disable gen-schema hook" This reverts commit 0e629af83302244916cdad2ad4e10ce37bb176a9. --- .github/workflows/ci.yaml | 4 ++-- .pre-commit-config.yaml | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fc45ec1f1..84577d1ab 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -64,8 +64,8 @@ jobs: poetry run pre-commit run pyright --all-files fi; - # - name: Generate schema (kpops schema) - # run: poetry run pre-commit run gen-schema --all-files --show-diff-on-failure + - name: Generate schema (kpops schema) + run: poetry run pre-commit run gen-schema --all-files --show-diff-on-failure - name: Generate CLI Usage docs (typer-cli) run: poetry run pre-commit run gen-docs-cli --all-files --show-diff-on-failure diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 04e870704..10a660058 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -29,14 +29,14 @@ repos: language: system types: [python] require_serial: true # run once for all files - # - id: gen-schema - # name: gen-schema - # entry: python hooks/gen_schema.py - # language: system - # types: [python] - # require_serial: true - # pass_filenames: false - # exclude: ^tests/.*snapshots/ + - id: gen-schema + name: gen-schema + entry: python hooks/gen_schema.py + language: system + types: [python] + require_serial: true + pass_filenames: false + exclude: ^tests/.*snapshots/ - id: gen-docs-cli name: gen-docs-cli entry: python hooks/gen_docs/gen_docs_cli_usage.py From 5af8684004f064721ffb80e93bbdba96e2c25e6e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:53:44 +0200 Subject: [PATCH 27/48] Revert "Temporarily disable schema command" This reverts commit b83c5a25bb458b9ce189e37821d5e42fd96e85e0. --- docs/docs/user/references/cli-commands.md | 34 ++++++++++ kpops/cli/main.py | 80 ++++++++++++----------- 2 files changed, 77 insertions(+), 37 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index 934e369c3..570563069 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -22,6 +22,7 @@ $ kpops [OPTIONS] COMMAND [ARGS]... * `init`: Initialize a new KPOps project. * `manifest`: Render final resource representation * `reset`: Reset pipeline steps +* `schema`: Generate JSON schema. ## `kpops clean` @@ -193,3 +194,36 @@ $ kpops reset [OPTIONS] PIPELINE_PATHS... * `--verbose / --no-verbose`: Enable verbose printing [default: no-verbose] * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially. [default: no-parallel] * `--help`: Show this message and exit. + +## `kpops schema` + +Generate JSON schema. + +The schemas can be used to enable support for KPOps files in a text editor. + +**Usage**: + +```console +$ kpops schema [OPTIONS] SCOPE:{pipeline|defaults|config} +``` + +**Arguments**: + +* `SCOPE:{pipeline|defaults|config}`: + Scope of the generated schema + + + + + pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. + + + + + config: Schema of KpopsConfig. [required] + +**Options**: + +* `--config DIRECTORY`: Path to the dir containing config.yaml files [env var: KPOPS_CONFIG_PATH; default: .] +* `--include-stock-components / --no-include-stock-components`: Include the built-in KPOps components. [default: include-stock-components] +* `--help`: Show this message and exit. diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 1562b43d2..395ab8e53 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -7,9 +7,15 @@ import kpops from kpops import __version__ +from kpops.api.file_type import KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import collect_pipeline_paths -from kpops.config import ENV_PREFIX +from kpops.config import ENV_PREFIX, KpopsConfig +from kpops.utils.gen_schema import ( + gen_config_schema, + gen_defaults_schema, + gen_pipeline_schema, +) from kpops.utils.yaml import print_yaml app = typer.Typer(pretty_exceptions_enable=False) @@ -112,42 +118,42 @@ def init( kpops.init(path, config_include_opt=config_include_opt) -# @app.command( -# help=""" -# Generate JSON schema. - -# The schemas can be used to enable support for KPOps files in a text editor. -# """ -# ) -# def schema( -# scope: KpopsFileType = typer.Argument( -# ..., -# show_default=False, -# help=""" -# Scope of the generated schema -# \n\n\n -# pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. -# \n\n\n -# config: Schema of KpopsConfig.""", -# ), -# config: Path = CONFIG_PATH_OPTION, -# include_stock_components: bool = typer.Option( -# default=True, help="Include the built-in KPOps components." -# ), -# ) -> None: -# match scope: -# case KpopsFileType.PIPELINE: -# kpops_config = KpopsConfig.create(config) -# gen_pipeline_schema( -# kpops_config.components_module, include_stock_components -# ) -# case KpopsFileType.DEFAULTS: -# kpops_config = KpopsConfig.create(config) -# gen_defaults_schema( -# kpops_config.components_module, include_stock_components -# ) -# case KpopsFileType.CONFIG: -# gen_config_schema() +@app.command( + help=""" + Generate JSON schema. + + The schemas can be used to enable support for KPOps files in a text editor. + """ +) +def schema( + scope: KpopsFileType = typer.Argument( + ..., + show_default=False, + help=""" + Scope of the generated schema + \n\n\n + pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. + \n\n\n + config: Schema of KpopsConfig.""", + ), + config: Path = CONFIG_PATH_OPTION, + include_stock_components: bool = typer.Option( + default=True, help="Include the built-in KPOps components." + ), +) -> None: + match scope: + case KpopsFileType.PIPELINE: + kpops_config = KpopsConfig.create(config) + gen_pipeline_schema( + kpops_config.components_module, include_stock_components + ) + case KpopsFileType.DEFAULTS: + kpops_config = KpopsConfig.create(config) + gen_defaults_schema( + kpops_config.components_module, include_stock_components + ) + case KpopsFileType.CONFIG: + gen_config_schema() @app.command( From af6997b0b444bfcb6838d377453bbddb3d46f13e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 20:59:43 +0200 Subject: [PATCH 28/48] Update schema generation --- docs/docs/user/references/cli-commands.md | 2 - kpops/cli/main.py | 16 +---- tests/cli/test_schema_generation.py | 87 ----------------------- 3 files changed, 3 insertions(+), 102 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index 570563069..173032d65 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -224,6 +224,4 @@ $ kpops schema [OPTIONS] SCOPE:{pipeline|defaults|config} **Options**: -* `--config DIRECTORY`: Path to the dir containing config.yaml files [env var: KPOPS_CONFIG_PATH; default: .] -* `--include-stock-components / --no-include-stock-components`: Include the built-in KPOps components. [default: include-stock-components] * `--help`: Show this message and exit. diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 395ab8e53..c84c4c403 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -10,7 +10,7 @@ from kpops.api.file_type import KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import collect_pipeline_paths -from kpops.config import ENV_PREFIX, KpopsConfig +from kpops.config import ENV_PREFIX from kpops.utils.gen_schema import ( gen_config_schema, gen_defaults_schema, @@ -136,22 +136,12 @@ def schema( \n\n\n config: Schema of KpopsConfig.""", ), - config: Path = CONFIG_PATH_OPTION, - include_stock_components: bool = typer.Option( - default=True, help="Include the built-in KPOps components." - ), ) -> None: match scope: case KpopsFileType.PIPELINE: - kpops_config = KpopsConfig.create(config) - gen_pipeline_schema( - kpops_config.components_module, include_stock_components - ) + gen_pipeline_schema() case KpopsFileType.DEFAULTS: - kpops_config = KpopsConfig.create(config) - gen_defaults_schema( - kpops_config.components_module, include_stock_components - ) + gen_defaults_schema() case KpopsFileType.CONFIG: gen_config_schema() diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index a6fe57380..a15028b5b 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -3,7 +3,6 @@ import json from abc import ABC, abstractmethod from pathlib import Path -from typing import TYPE_CHECKING import pytest from pydantic import ConfigDict, Field @@ -14,9 +13,6 @@ from kpops.components import PipelineComponent from kpops.utils.docstring import describe_attr -if TYPE_CHECKING: - from pytest_snapshot.plugin import Snapshot - RESOURCE_PATH = Path(__file__).parent / "resources" @@ -76,7 +72,6 @@ class SubPipelineComponentCorrectDocstr(SubPipelineComponent): ) -@pytest.mark.skip() @pytest.mark.filterwarnings( "ignore:handlers", "ignore:config", "ignore:enrich", "ignore:validate" ) @@ -87,86 +82,6 @@ def stock_components(self) -> list[type[PipelineComponent]]: registry.discover_components() return list(registry._classes.values()) - def test_gen_pipeline_schema_no_modules(self): - with pytest.raises( - RuntimeError, match="^No components are provided, no schema is generated.$" - ): - runner.invoke( - app, - [ - "schema", - "pipeline", - "--no-include-stock-components", - "--config", - str(RESOURCE_PATH / "no_module"), - ], - catch_exceptions=False, - ) - - def test_gen_pipeline_schema_no_components(self): - with pytest.raises(RuntimeError, match="^No valid components found.$"): - runner.invoke( - app, - [ - "schema", - "pipeline", - "--no-include-stock-components", - "--config", - str(RESOURCE_PATH / "empty_module"), - ], - catch_exceptions=False, - ) - - def test_gen_pipeline_schema_only_stock_module(self): - result = runner.invoke( - app, - [ - "schema", - "pipeline", - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - assert result.stdout - - result = runner.invoke( - app, - [ - "schema", - "pipeline", - "--include-stock-components", - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - assert result.stdout - - def test_gen_pipeline_schema_only_custom_module( - self, snapshot: Snapshot, stock_components: list[type[PipelineComponent]] - ): - result = runner.invoke( - app, - [ - "schema", - "pipeline", - "--no-include-stock-components", - "--config", - str(RESOURCE_PATH), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, "schema.json") - schema = json.loads(result.stdout) - assert schema["title"] == "PipelineSchema" - assert set(schema["items"]["discriminator"]["mapping"].keys()).isdisjoint( - {component.type for component in stock_components} - ) - def test_gen_pipeline_schema_stock_and_custom_module(self): result = runner.invoke( app, @@ -186,8 +101,6 @@ def test_gen_defaults_schema(self, stock_components: list[type[PipelineComponent [ "schema", "defaults", - "--config", - str(RESOURCE_PATH / "no_module"), ], catch_exceptions=False, ) From d6d6749a75a73b3bbf49262384bdbe12f9b915b7 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 23:42:55 +0200 Subject: [PATCH 29/48] Try namespace package --- hooks/gen_docs/gen_docs_components.py | 5 ++- kpops/__init__.py | 14 ------- kpops/api/__init__.py | 13 +++---- kpops/api/logs.py | 2 +- kpops/api/options.py | 2 +- kpops/api/registry.py | 3 +- kpops/cli/main.py | 5 +-- kpops/components/__init__.py | 27 ------------- kpops/components/base_components/__init__.py | 19 ++++++++++ kpops/components/base_components/kafka_app.py | 2 +- .../components/streams_bootstrap/__init__.py | 38 ++++--------------- .../producer/producer_app.py | 2 +- .../streams_bootstrap/streams/streams_app.py | 9 ++--- kpops/utils/gen_schema.py | 2 +- tests/api/test_registry.py | 16 ++++---- tests/cli/test_init.py | 2 +- tests/cli/test_schema_generation.py | 2 +- tests/components/test_kafka_sink_connector.py | 6 ++- tests/components/test_producer_app.py | 6 ++- tests/components/test_streams_app.py | 2 +- tests/components/test_streams_bootstrap.py | 2 +- tests/pipeline/test_components/components.py | 10 ++--- .../components.py | 10 ++--- tests/pipeline/test_example.py | 2 +- tests/pipeline/test_generate.py | 33 ++++------------ tests/pipeline/test_manifest.py | 2 +- tests/pipeline/test_pipeline.py | 2 +- 27 files changed, 86 insertions(+), 152 deletions(-) delete mode 100644 kpops/__init__.py delete mode 100644 kpops/components/__init__.py diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index b6c733267..ed28dfe6e 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -9,7 +9,10 @@ from hooks import ROOT from kpops.api.registry import Registry -from kpops.components import KafkaConnector, PipelineComponent +from kpops.components.base_components.kafka_connector import KafkaConnector +from kpops.components.base_components.pipeline_component import ( + PipelineComponent, +) from kpops.utils.colorify import redify, yellowify from kpops.utils.pydantic import issubclass_patched from kpops.utils.yaml import load_yaml_file diff --git a/kpops/__init__.py b/kpops/__init__.py deleted file mode 100644 index b0b3c6651..000000000 --- a/kpops/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -__version__ = "6.0.1" - -# export public API functions -from kpops.api import clean, deploy, destroy, generate, init, manifest, reset - -__all__ = ( - "generate", - "manifest", - "deploy", - "destroy", - "reset", - "clean", - "init", -) diff --git a/kpops/api/__init__.py b/kpops/api/__init__.py index aa7133fa2..5a3a81102 100644 --- a/kpops/api/__init__.py +++ b/kpops/api/__init__.py @@ -4,7 +4,6 @@ from pathlib import Path from typing import TYPE_CHECKING -import kpops from kpops.api.logs import log, log_action from kpops.api.options import FilterType from kpops.api.registry import Registry @@ -23,8 +22,8 @@ from kpops.utils.cli_commands import init_project if TYPE_CHECKING: - from kpops.components import PipelineComponent from kpops.components.base_components.models.resource import Resource + from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.config import KpopsConfig @@ -68,7 +67,7 @@ def manifest( environment: str | None = None, verbose: bool = False, ) -> list[Resource]: - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -95,7 +94,7 @@ def deploy( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -131,7 +130,7 @@ def destroy( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -169,7 +168,7 @@ def reset( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, @@ -206,7 +205,7 @@ def clean( verbose: bool = True, parallel: bool = False, ): - pipeline = kpops.generate( + pipeline = generate( pipeline_path=pipeline_path, dotenv=dotenv, config=config, diff --git a/kpops/api/logs.py b/kpops/api/logs.py index e9a833aba..979b5a36c 100644 --- a/kpops/api/logs.py +++ b/kpops/api/logs.py @@ -6,7 +6,7 @@ import typer if TYPE_CHECKING: - from kpops.components import PipelineComponent + from kpops.components.base_components.pipeline_component import PipelineComponent class CustomFormatter(logging.Formatter): diff --git a/kpops/api/options.py b/kpops/api/options.py index dc116bd35..22fda2542 100644 --- a/kpops/api/options.py +++ b/kpops/api/options.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from kpops.components import PipelineComponent + from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.pipeline import ComponentFilterPredicate diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 592f481d2..7a5b4401d 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -11,14 +11,13 @@ from types import ModuleType from typing import TYPE_CHECKING, TypeVar -from kpops import __name__ from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent if TYPE_CHECKING: from collections.abc import Iterator -KPOPS_MODULE = __name__ + "." +KPOPS_MODULE = "kpops." T = TypeVar("T") ClassDict = dict[str, type[T]] # type -> class diff --git a/kpops/cli/main.py b/kpops/cli/main.py index c84c4c403..3e669df54 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -5,8 +5,7 @@ import typer -import kpops -from kpops import __version__ +import kpops.api as kpops from kpops.api.file_type import KpopsFileType from kpops.api.options import FilterType from kpops.cli.utils import collect_pipeline_paths @@ -306,7 +305,7 @@ def clean( def version_callback(show_version: bool) -> None: if show_version: - typer.echo(f"KPOps {__version__}") + # typer.echo(f"KPOps {__version__}") # FIXME: read version from package raise typer.Exit diff --git a/kpops/components/__init__.py b/kpops/components/__init__.py deleted file mode 100644 index 3800b16d5..000000000 --- a/kpops/components/__init__.py +++ /dev/null @@ -1,27 +0,0 @@ -from kpops.components.base_components.helm_app import HelmApp -from kpops.components.base_components.kafka_app import KafkaApp -from kpops.components.base_components.kafka_connector import ( - KafkaConnector, - KafkaSinkConnector, - KafkaSourceConnector, -) -from kpops.components.base_components.kubernetes_app import KubernetesApp -from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.streams_bootstrap import StreamsBootstrap -from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp -from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp - -__all__ = ( - "HelmApp", - "KafkaApp", - "KafkaConnector", - "KafkaSinkConnector", - "KafkaSourceConnector", - "KubernetesApp", - "StreamsBootstrap", - "ProducerApp", - "StreamsApp", - "PipelineComponent", - "StreamsApp", - "ProducerApp", -) diff --git a/kpops/components/base_components/__init__.py b/kpops/components/base_components/__init__.py index e69de29bb..ff94dde1f 100644 --- a/kpops/components/base_components/__init__.py +++ b/kpops/components/base_components/__init__.py @@ -0,0 +1,19 @@ +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp +from kpops.components.base_components.kafka_connector import ( + KafkaConnector, + KafkaSinkConnector, + KafkaSourceConnector, +) +from kpops.components.base_components.kubernetes_app import KubernetesApp +from kpops.components.base_components.pipeline_component import PipelineComponent + +__all__ = ( + "HelmApp", + "KafkaApp", + "KafkaConnector", + "KafkaSinkConnector", + "KafkaSourceConnector", + "KubernetesApp", + "PipelineComponent", +) diff --git a/kpops/components/base_components/kafka_app.py b/kpops/components/base_components/kafka_app.py index f52ca6e78..0ab4806b9 100644 --- a/kpops/components/base_components/kafka_app.py +++ b/kpops/components/base_components/kafka_app.py @@ -13,7 +13,7 @@ from kpops.components.base_components.helm_app import HelmAppValues from kpops.components.base_components.models.topic import KafkaTopic, KafkaTopicStr from kpops.components.base_components.pipeline_component import PipelineComponent -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.utils.docstring import describe_attr from kpops.utils.pydantic import ( CamelCaseConfigModel, diff --git a/kpops/components/streams_bootstrap/__init__.py b/kpops/components/streams_bootstrap/__init__.py index 1b02b091b..b4eb34b2f 100644 --- a/kpops/components/streams_bootstrap/__init__.py +++ b/kpops/components/streams_bootstrap/__init__.py @@ -1,31 +1,9 @@ -from abc import ABC - -from pydantic import Field - -from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig -from kpops.components.base_components.helm_app import HelmApp -from kpops.utils.docstring import describe_attr - -STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( - repository_name="bakdata-streams-bootstrap", - url="https://bakdata.github.io/streams-bootstrap/", +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp + +__all__ = ( + "StreamsBootstrap", + "StreamsApp", + "ProducerApp", ) -STREAMS_BOOTSTRAP_VERSION = "2.9.0" - - -class StreamsBootstrap(HelmApp, ABC): - """Base for components with a streams-bootstrap Helm chart. - - :param repo_config: Configuration of the Helm chart repo to be used for - deploying the component, defaults to streams-bootstrap Helm repo - :param version: Helm chart version, defaults to "2.9.0" - """ - - repo_config: HelmRepoConfig = Field( - default=STREAMS_BOOTSTRAP_HELM_REPO, - description=describe_attr("repo_config", __doc__), - ) - version: str | None = Field( - default=STREAMS_BOOTSTRAP_VERSION, - description=describe_attr("version", __doc__), - ) diff --git a/kpops/components/streams_bootstrap/producer/producer_app.py b/kpops/components/streams_bootstrap/producer/producer_app.py index a9c06f37e..9674dd7da 100644 --- a/kpops/components/streams_bootstrap/producer/producer_app.py +++ b/kpops/components/streams_bootstrap/producer/producer_app.py @@ -12,7 +12,7 @@ OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.producer.model import ProducerAppValues from kpops.utils.docstring import describe_attr diff --git a/kpops/components/streams_bootstrap/streams/streams_app.py b/kpops/components/streams_bootstrap/streams/streams_app.py index 9bd5d87c5..979c3c4c9 100644 --- a/kpops/components/streams_bootstrap/streams/streams_app.py +++ b/kpops/components/streams_bootstrap/streams/streams_app.py @@ -5,13 +5,10 @@ from typing_extensions import override from kpops.component_handlers.kubernetes.pvc_handler import PVCHandler -from kpops.components import HelmApp -from kpops.components.base_components.kafka_app import ( - KafkaApp, - KafkaAppCleaner, -) +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp, KafkaAppCleaner from kpops.components.base_components.models.topic import KafkaTopic -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.components.streams_bootstrap.app_type import AppType from kpops.components.streams_bootstrap.streams.model import ( StreamsAppValues, diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index 6cf326ed0..4535c05e7 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -21,7 +21,7 @@ ) from kpops.api.registry import Registry -from kpops.components import ( +from kpops.components.base_components.pipeline_component import ( PipelineComponent, ) from kpops.config import KpopsConfig diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index 46e62f17f..e582b2c18 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -8,18 +8,18 @@ from kpops.api.exception import ClassNotFoundError from kpops.api.registry import Registry, _find_classes, _iter_namespace, find_class from kpops.component_handlers.schema_handler.schema_provider import SchemaProvider -from kpops.components import ( - HelmApp, - KafkaApp, +from kpops.components.base_components.helm_app import HelmApp +from kpops.components.base_components.kafka_app import KafkaApp +from kpops.components.base_components.kafka_connector import ( KafkaConnector, KafkaSinkConnector, KafkaSourceConnector, - KubernetesApp, - PipelineComponent, - ProducerApp, - StreamsApp, - StreamsBootstrap, ) +from kpops.components.base_components.kubernetes_app import KubernetesApp +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.common.streams_bootstrap import StreamsBootstrap +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp from tests.cli.resources.custom_module import CustomSchemaProvider diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 8d4790d23..3109d16fc 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -3,7 +3,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.cli.main import app from kpops.utils.cli_commands import create_config diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index a15028b5b..dc30f0543 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -10,7 +10,7 @@ from kpops.api.registry import Registry from kpops.cli.main import app -from kpops.components import PipelineComponent +from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.docstring import describe_attr RESOURCE_PATH = Path(__file__).parent / "resources" diff --git a/tests/components/test_kafka_sink_connector.py b/tests/components/test_kafka_sink_connector.py index 51b30a61d..1f9747280 100644 --- a/tests/components/test_kafka_sink_connector.py +++ b/tests/components/test_kafka_sink_connector.py @@ -12,8 +12,10 @@ KafkaConnectorConfig, KafkaConnectorType, ) -from kpops.components import KafkaSinkConnector -from kpops.components.base_components.kafka_connector import KafkaConnectorResetter +from kpops.components.base_components.kafka_connector import ( + KafkaConnectorResetter, + KafkaSinkConnector, +) from kpops.components.base_components.models.from_section import ( FromSection, FromTopic, diff --git a/tests/components/test_producer_app.py b/tests/components/test_producer_app.py index 4f7184ead..346913e57 100644 --- a/tests/components/test_producer_app.py +++ b/tests/components/test_producer_app.py @@ -7,13 +7,15 @@ from kpops.component_handlers import ComponentHandlers from kpops.component_handlers.helm_wrapper.model import HelmUpgradeInstallFlags from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components import ProducerApp from kpops.components.base_components.models.topic import ( KafkaTopic, OutputTopicTypes, TopicConfig, ) -from kpops.components.streams_bootstrap.producer.producer_app import ProducerAppCleaner +from kpops.components.streams_bootstrap.producer.producer_app import ( + ProducerApp, + ProducerAppCleaner, +) from kpops.config import KpopsConfig, TopicNameConfig from tests.components import PIPELINE_BASE_DIR diff --git a/tests/components/test_streams_app.py b/tests/components/test_streams_app.py index cb340174a..a72328d5e 100644 --- a/tests/components/test_streams_app.py +++ b/tests/components/test_streams_app.py @@ -12,7 +12,6 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components import StreamsApp from kpops.components.base_components.models import TopicName from kpops.components.base_components.models.to_section import ( ToSection, @@ -27,6 +26,7 @@ StreamsAppAutoScaling, ) from kpops.components.streams_bootstrap.streams.streams_app import ( + StreamsApp, StreamsAppCleaner, ) from kpops.config import KpopsConfig, TopicNameConfig diff --git a/tests/components/test_streams_bootstrap.py b/tests/components/test_streams_bootstrap.py index a82fca8a9..d014d2b32 100644 --- a/tests/components/test_streams_bootstrap.py +++ b/tests/components/test_streams_bootstrap.py @@ -10,7 +10,7 @@ HelmUpgradeInstallFlags, ) from kpops.component_handlers.helm_wrapper.utils import create_helm_release_name -from kpops.components.streams_bootstrap import StreamsBootstrap +from kpops.components.common.streams_bootstrap import StreamsBootstrap from kpops.config import KpopsConfig from tests.components import PIPELINE_BASE_DIR diff --git a/tests/pipeline/test_components/components.py b/tests/pipeline/test_components/components.py index 36f24f938..4502ea69e 100644 --- a/tests/pipeline/test_components/components.py +++ b/tests/pipeline/test_components/components.py @@ -5,17 +5,15 @@ Schema, SchemaProvider, ) -from kpops.components import ( - KafkaSinkConnector, - PipelineComponent, - ProducerApp, - StreamsApp, -) +from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.models import ModelName, ModelVersion, TopicName from kpops.components.base_components.models.to_section import ( ToSection, ) from kpops.components.base_components.models.topic import OutputTopicTypes, TopicConfig +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp class ScheduledProducer(ProducerApp): ... diff --git a/tests/pipeline/test_components_without_schema_handler/components.py b/tests/pipeline/test_components_without_schema_handler/components.py index 1d54a9f7a..9646e569b 100644 --- a/tests/pipeline/test_components_without_schema_handler/components.py +++ b/tests/pipeline/test_components_without_schema_handler/components.py @@ -1,13 +1,11 @@ from typing_extensions import override from kpops.component_handlers.kafka_connect.model import KafkaConnectorConfig -from kpops.components import ( - KafkaSinkConnector, - PipelineComponent, - ProducerApp, - StreamsApp, -) +from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.models.topic import OutputTopicTypes +from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.components.streams_bootstrap.producer.producer_app import ProducerApp +from kpops.components.streams_bootstrap.streams.streams_app import StreamsApp class ScheduledProducer(ProducerApp): ... diff --git a/tests/pipeline/test_example.py b/tests/pipeline/test_example.py index 1101b5795..7a03ae2ee 100644 --- a/tests/pipeline/test_example.py +++ b/tests/pipeline/test_example.py @@ -5,7 +5,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops runner = CliRunner() diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index a209071e3..f0daf13d7 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -9,11 +9,12 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.api.exception import ParsingException, ValidationError from kpops.api.file_type import KpopsFileType from kpops.cli.main import FilterType, app -from kpops.components import KafkaSinkConnector, PipelineComponent +from kpops.components.base_components.kafka_connector import KafkaSinkConnector +from kpops.components.base_components.pipeline_component import PipelineComponent PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() @@ -66,32 +67,12 @@ def test_python_api_filter_exclude(self, log_info: MagicMock): ) def test_load_pipeline(self, snapshot: Snapshot): - result = runner.invoke( - app, - [ - "generate", - str(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, PIPELINE_YAML) + pipeline = kpops.generate(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML) + snapshot.assert_match(pipeline.to_yaml(), PIPELINE_YAML) def test_load_pipeline_with_folder_path(self, snapshot: Snapshot): - result = runner.invoke( - app, - [ - "generate", - str(RESOURCE_PATH / "pipeline-folders"), - ], - catch_exceptions=False, - ) - - assert result.exit_code == 0, result.stdout - - snapshot.assert_match(result.stdout, "pipeline.yaml") + pipeline = kpops.generate(RESOURCE_PATH / "pipeline-folders") + snapshot.assert_match(pipeline.to_yaml(), "pipeline.yaml") def test_load_pipeline_with_multiple_pipeline_paths(self, snapshot: Snapshot): path_1 = RESOURCE_PATH / "pipeline-folders/pipeline-1/pipeline.yaml" diff --git a/tests/pipeline/test_manifest.py b/tests/pipeline/test_manifest.py index 445e528cc..45f3e3a94 100644 --- a/tests/pipeline/test_manifest.py +++ b/tests/pipeline/test_manifest.py @@ -7,7 +7,7 @@ from pytest_snapshot.plugin import Snapshot from typer.testing import CliRunner -import kpops +import kpops.api as kpops from kpops.cli.main import app from kpops.component_handlers.helm_wrapper.helm import Helm from kpops.component_handlers.helm_wrapper.model import HelmConfig, Version diff --git a/tests/pipeline/test_pipeline.py b/tests/pipeline/test_pipeline.py index b5c741ad0..61d338dcf 100644 --- a/tests/pipeline/test_pipeline.py +++ b/tests/pipeline/test_pipeline.py @@ -7,9 +7,9 @@ from kpops.component_handlers import ( ComponentHandlers, ) -from kpops.components import PipelineComponent from kpops.components.base_components.models.from_section import FromSection from kpops.components.base_components.models.to_section import ToSection +from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.pipeline import Pipeline PREFIX = "example-prefix-" From 24d0e516d776d0c369c46cdcf233006e764bc68d Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 23:55:19 +0200 Subject: [PATCH 30/48] Fix Ruff lint --- kpops/{config.py => config/__init__.py} | 0 kpops/{pipeline.py => pipeline/__init__.py} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename kpops/{config.py => config/__init__.py} (100%) rename kpops/{pipeline.py => pipeline/__init__.py} (100%) diff --git a/kpops/config.py b/kpops/config/__init__.py similarity index 100% rename from kpops/config.py rename to kpops/config/__init__.py diff --git a/kpops/pipeline.py b/kpops/pipeline/__init__.py similarity index 100% rename from kpops/pipeline.py rename to kpops/pipeline/__init__.py From c7ec556a0d5a183baeda10f145b1a06a31e365b2 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 26 Jun 2024 23:58:59 +0200 Subject: [PATCH 31/48] Add missing files --- kpops/components/common/__init__.py | 0 kpops/components/common/streams_bootstrap.py | 31 ++++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 kpops/components/common/__init__.py create mode 100644 kpops/components/common/streams_bootstrap.py diff --git a/kpops/components/common/__init__.py b/kpops/components/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/kpops/components/common/streams_bootstrap.py b/kpops/components/common/streams_bootstrap.py new file mode 100644 index 000000000..1b02b091b --- /dev/null +++ b/kpops/components/common/streams_bootstrap.py @@ -0,0 +1,31 @@ +from abc import ABC + +from pydantic import Field + +from kpops.component_handlers.helm_wrapper.model import HelmRepoConfig +from kpops.components.base_components.helm_app import HelmApp +from kpops.utils.docstring import describe_attr + +STREAMS_BOOTSTRAP_HELM_REPO = HelmRepoConfig( + repository_name="bakdata-streams-bootstrap", + url="https://bakdata.github.io/streams-bootstrap/", +) +STREAMS_BOOTSTRAP_VERSION = "2.9.0" + + +class StreamsBootstrap(HelmApp, ABC): + """Base for components with a streams-bootstrap Helm chart. + + :param repo_config: Configuration of the Helm chart repo to be used for + deploying the component, defaults to streams-bootstrap Helm repo + :param version: Helm chart version, defaults to "2.9.0" + """ + + repo_config: HelmRepoConfig = Field( + default=STREAMS_BOOTSTRAP_HELM_REPO, + description=describe_attr("repo_config", __doc__), + ) + version: str | None = Field( + default=STREAMS_BOOTSTRAP_VERSION, + description=describe_attr("version", __doc__), + ) From 0279d769447270fff7ef686cae549e0f16a6a79e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 00:04:58 +0200 Subject: [PATCH 32/48] Update test --- tests/api/test_registry.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index e582b2c18..df5223a71 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -41,6 +41,7 @@ def test_iter_namespace(): components_module = importlib.import_module("kpops.components") assert [module.__name__ for module in _iter_namespace(components_module)] == [ "kpops.components.base_components", + "kpops.components.common", "kpops.components.streams_bootstrap", "kpops.components.test_components", ] @@ -51,6 +52,7 @@ def test_iter_component_modules(): assert [module.__name__ for module in Registry.iter_component_modules()] == [ "kpops.components", "kpops.components.base_components", + "kpops.components.common", "kpops.components.streams_bootstrap", "kpops.components.test_components", ] From d22e789a126ce4b3008091f78f936fdf7088c3d5 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 00:07:57 +0200 Subject: [PATCH 33/48] Revert mistake --- tests/pipeline/test_generate.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index f0daf13d7..58209e384 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -67,12 +67,32 @@ def test_python_api_filter_exclude(self, log_info: MagicMock): ) def test_load_pipeline(self, snapshot: Snapshot): - pipeline = kpops.generate(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML) - snapshot.assert_match(pipeline.to_yaml(), PIPELINE_YAML) + result = runner.invoke( + app, + [ + "generate", + str(RESOURCE_PATH / "first-pipeline" / PIPELINE_YAML), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.stdout + + snapshot.assert_match(result.stdout, PIPELINE_YAML) def test_load_pipeline_with_folder_path(self, snapshot: Snapshot): - pipeline = kpops.generate(RESOURCE_PATH / "pipeline-folders") - snapshot.assert_match(pipeline.to_yaml(), "pipeline.yaml") + result = runner.invoke( + app, + [ + "generate", + str(RESOURCE_PATH / "pipeline-folders"), + ], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.stdout + + snapshot.assert_match(result.stdout, "pipeline.yaml") def test_load_pipeline_with_multiple_pipeline_paths(self, snapshot: Snapshot): path_1 = RESOURCE_PATH / "pipeline-folders/pipeline-1/pipeline.yaml" From 0fe22e972a44e4bc947aafdc482b32ee7ad286a3 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 00:19:20 +0200 Subject: [PATCH 34/48] Improve CLI help --- docs/docs/user/references/cli-commands.md | 7 +++++-- kpops/api/file_type.py | 5 +++++ kpops/cli/main.py | 16 ++++++++++------ kpops/cli/utils.py | 4 +--- tests/components/test_base_defaults_component.py | 6 +----- tests/pipeline/test_generate.py | 4 +--- 6 files changed, 23 insertions(+), 19 deletions(-) diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md index 173032d65..0a187f139 100644 --- a/docs/docs/user/references/cli-commands.md +++ b/docs/docs/user/references/cli-commands.md @@ -215,12 +215,15 @@ $ kpops schema [OPTIONS] SCOPE:{pipeline|defaults|config} - pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. + - pipeline: Schema of PipelineComponents for KPOps pipeline.yaml + - defaults: Schema of PipelineComponents for KPOps defaults.yaml + + - config: Schema of KpopsConfig. [required] + - config: Schema for KPOps config.yaml [required] **Options**: diff --git a/kpops/api/file_type.py b/kpops/api/file_type.py index c08fce987..3e170be96 100644 --- a/kpops/api/file_type.py +++ b/kpops/api/file_type.py @@ -33,3 +33,8 @@ def as_yaml_file(self, prefix: str = "", suffix: str = "") -> str: 'pre_pipeline_suf.yaml' """ return prefix + self.value + suffix + FILE_EXTENSION + + +PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() +DEFAULTS_YAML = KpopsFileType.DEFAULTS.as_yaml_file() +CONFIG_YAML = KpopsFileType.CONFIG.as_yaml_file() diff --git a/kpops/cli/main.py b/kpops/cli/main.py index 3e669df54..d33712b3e 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -6,9 +6,11 @@ import typer import kpops.api as kpops -from kpops.api.file_type import KpopsFileType +from kpops.api.file_type import CONFIG_YAML, DEFAULTS_YAML, PIPELINE_YAML, KpopsFileType from kpops.api.options import FilterType -from kpops.cli.utils import collect_pipeline_paths +from kpops.cli.utils import ( + collect_pipeline_paths, +) from kpops.config import ENV_PREFIX from kpops.utils.gen_schema import ( gen_config_schema, @@ -128,12 +130,14 @@ def schema( scope: KpopsFileType = typer.Argument( ..., show_default=False, - help=""" + help=f""" Scope of the generated schema \n\n\n - pipeline: Schema of PipelineComponents. Includes the built-in KPOps components by default. To include custom components, provide components module in config. - \n\n\n - config: Schema of KpopsConfig.""", + - {KpopsFileType.PIPELINE.value}: Schema of PipelineComponents for KPOps {PIPELINE_YAML} + \n\n + - {KpopsFileType.DEFAULTS.value}: Schema of PipelineComponents for KPOps {DEFAULTS_YAML} + \n\n + - {KpopsFileType.CONFIG.value}: Schema for KPOps {CONFIG_YAML}""", ), ) -> None: match scope: diff --git a/kpops/cli/utils.py b/kpops/cli/utils.py index f4a04bcbb..16a8af563 100644 --- a/kpops/cli/utils.py +++ b/kpops/cli/utils.py @@ -3,9 +3,7 @@ from collections.abc import Iterable, Iterator from pathlib import Path -from kpops.api.file_type import KpopsFileType - -PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() +from kpops.api.file_type import PIPELINE_YAML def collect_pipeline_paths(pipeline_paths: Iterable[Path]) -> Iterator[Path]: diff --git a/tests/components/test_base_defaults_component.py b/tests/components/test_base_defaults_component.py index ab21ef68e..f8fe12eec 100644 --- a/tests/components/test_base_defaults_component.py +++ b/tests/components/test_base_defaults_component.py @@ -6,7 +6,7 @@ import pydantic import pytest -from kpops.api.file_type import KpopsFileType +from kpops.api.file_type import DEFAULTS_YAML, PIPELINE_YAML, KpopsFileType from kpops.component_handlers import ComponentHandlers from kpops.components.base_components.base_defaults_component import ( BaseDefaultsComponent, @@ -17,10 +17,6 @@ from kpops.utils.environment import ENV from tests.components import PIPELINE_BASE_DIR, RESOURCES_PATH -PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() - -DEFAULTS_YAML = KpopsFileType.DEFAULTS.as_yaml_file() - class Parent(BaseDefaultsComponent): __test__ = False diff --git a/tests/pipeline/test_generate.py b/tests/pipeline/test_generate.py index 58209e384..07da70082 100644 --- a/tests/pipeline/test_generate.py +++ b/tests/pipeline/test_generate.py @@ -11,13 +11,11 @@ import kpops.api as kpops from kpops.api.exception import ParsingException, ValidationError -from kpops.api.file_type import KpopsFileType +from kpops.api.file_type import PIPELINE_YAML, KpopsFileType from kpops.cli.main import FilterType, app from kpops.components.base_components.kafka_connector import KafkaSinkConnector from kpops.components.base_components.pipeline_component import PipelineComponent -PIPELINE_YAML = KpopsFileType.PIPELINE.as_yaml_file() - runner = CliRunner() RESOURCE_PATH = Path(__file__).parent / "resources" From e454b1acd13dc90c59fe5a472d023c6e128c20b4 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 13:38:38 +0200 Subject: [PATCH 35/48] Add draft for migration --- docs/docs/user/migration-guide/v6-v7.md | 8 ++++++++ docs/mkdocs.yml | 1 + 2 files changed, 9 insertions(+) create mode 100644 docs/docs/user/migration-guide/v6-v7.md diff --git a/docs/docs/user/migration-guide/v6-v7.md b/docs/docs/user/migration-guide/v6-v7.md new file mode 100644 index 000000000..74bf86c1e --- /dev/null +++ b/docs/docs/user/migration-guide/v6-v7.md @@ -0,0 +1,8 @@ +# Migrate from V6 to V7 + +```diff +-import kpops ++import kpops.api as kpops +``` + +TODO diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml index f4e3a8ba6..16390f472 100644 --- a/docs/mkdocs.yml +++ b/docs/mkdocs.yml @@ -114,6 +114,7 @@ nav: - Migrate from v3 to v4: user/migration-guide/v3-v4.md - Migrate from v4 to v5: user/migration-guide/v4-v5.md - Migrate from v5 to v6: user/migration-guide/v5-v6.md + - Migrate from v6 to v7: user/migration-guide/v6-v7.md - CLI usage: user/references/cli-commands.md - Editor integration: user/references/editor-integration.md - CI integration: From 6809be330d245813f0b02fd7f099a25bedf5498a Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 13:57:00 +0200 Subject: [PATCH 36/48] Add components property to registry --- hooks/gen_docs/gen_docs_components.py | 2 +- kpops/api/registry.py | 4 ++++ kpops/utils/gen_schema.py | 2 +- tests/cli/test_schema_generation.py | 10 ++-------- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/hooks/gen_docs/gen_docs_components.py b/hooks/gen_docs/gen_docs_components.py index ed28dfe6e..e5e03062d 100644 --- a/hooks/gen_docs/gen_docs_components.py +++ b/hooks/gen_docs/gen_docs_components.py @@ -39,7 +39,7 @@ (PATH_DOCS_RESOURCES / "pipeline-defaults/headers").iterdir(), ) -KPOPS_COMPONENTS = tuple(registry._classes.values()) +KPOPS_COMPONENTS = tuple(registry.components) KPOPS_COMPONENTS_SECTIONS = { component.type: [ field_name diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 7a5b4401d..90c0e81a9 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -32,6 +32,10 @@ class Registry: _classes: ClassDict[PipelineComponent] = field(default_factory=dict, init=False) + @property + def components(self) -> Iterator[type[PipelineComponent]]: + yield from self._classes.values() + def discover_components(self) -> None: """Discover first- and third-party KPOps components. diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index 4535c05e7..a8413a663 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -34,7 +34,7 @@ class MultiComponentGenerateJsonSchema(GenerateJsonSchema): ... registry = Registry() registry.discover_components() -COMPONENTS = tuple(registry._classes.values()) +COMPONENTS = tuple(registry.components) def print_schema(model: type[BaseModel]) -> None: diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index dc30f0543..802bb00a5 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -8,10 +8,10 @@ from pydantic import ConfigDict, Field from typer.testing import CliRunner -from kpops.api.registry import Registry from kpops.cli.main import app from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.utils.docstring import describe_attr +from kpops.utils.gen_schema import COMPONENTS RESOURCE_PATH = Path(__file__).parent / "resources" @@ -76,12 +76,6 @@ class SubPipelineComponentCorrectDocstr(SubPipelineComponent): "ignore:handlers", "ignore:config", "ignore:enrich", "ignore:validate" ) class TestGenSchema: - @pytest.fixture - def stock_components(self) -> list[type[PipelineComponent]]: - registry = Registry() - registry.discover_components() - return list(registry._classes.values()) - def test_gen_pipeline_schema_stock_and_custom_module(self): result = runner.invoke( app, @@ -109,7 +103,7 @@ def test_gen_defaults_schema(self, stock_components: list[type[PipelineComponent assert result.stdout schema = json.loads(result.stdout) assert schema["title"] == "DefaultsSchema" - assert schema["required"] == [component.type for component in stock_components] + assert schema["required"] == [component.type for component in COMPONENTS] def test_gen_config_schema(self): result = runner.invoke( From aa30817e39e954282ff441d778a92161d0c2b6d8 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 14:03:17 +0200 Subject: [PATCH 37/48] Cleanup & fix test --- kpops/utils/gen_schema.py | 15 +++------------ tests/cli/test_schema_generation.py | 2 +- 2 files changed, 4 insertions(+), 13 deletions(-) diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index a8413a663..88a7c42f1 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -48,8 +48,8 @@ def _is_valid_component( ) -> bool: """Check whether a PipelineComponent subclass has a valid definition for the schema generation. - :param defined_component_types: types defined so far :param component: component type to be validated + :param allow_abstract: whether to include abstract components marked as ABC :return: Whether component is valid for schema generation """ if not allow_abstract and ( @@ -61,13 +61,7 @@ def _is_valid_component( def gen_pipeline_schema() -> None: - """Generate a json schema from the models of pipeline components. - - :param components_module: Python module. Only the classes that inherit from - PipelineComponent will be considered., defaults to None - :param include_stock_components: Whether to include the stock components, - defaults to True - """ + """Generate a JSON schema from the models of pipeline components.""" components = [ component for component in COMPONENTS if _is_valid_component(component, False) ] @@ -103,10 +97,7 @@ class PipelineSchema(RootModel): print_schema(PipelineSchema) -def gen_defaults_schema( - components_module: str | None = None, include_stock_components: bool = True -) -> None: - # components = find_components(components_module, include_stock_components, True) +def gen_defaults_schema() -> None: components = [ component for component in COMPONENTS if _is_valid_component(component, True) ] diff --git a/tests/cli/test_schema_generation.py b/tests/cli/test_schema_generation.py index 802bb00a5..81f975b4c 100644 --- a/tests/cli/test_schema_generation.py +++ b/tests/cli/test_schema_generation.py @@ -89,7 +89,7 @@ def test_gen_pipeline_schema_stock_and_custom_module(self): assert result.exit_code == 0, result.stdout assert result.stdout - def test_gen_defaults_schema(self, stock_components: list[type[PipelineComponent]]): + def test_gen_defaults_schema(self): result = runner.invoke( app, [ From 4bc8ae5585722be494e136f66b45359b7b4beff4 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 17:27:38 +0200 Subject: [PATCH 38/48] Cosmetic --- kpops/utils/gen_schema.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kpops/utils/gen_schema.py b/kpops/utils/gen_schema.py index 88a7c42f1..539e2cf3a 100644 --- a/kpops/utils/gen_schema.py +++ b/kpops/utils/gen_schema.py @@ -68,14 +68,14 @@ def gen_pipeline_schema() -> None: # re-assign component type as Literal to work as discriminator for component in components: component.model_fields["type"] = FieldInfo( - annotation=Literal[component.type], # type:ignore[valid-type] + annotation=Literal[component.type], # type: ignore[valid-type] default=component.type, ) - core_schema: DefinitionsSchema = component.__pydantic_core_schema__ # pyright:ignore[reportAssignmentType] + core_schema: DefinitionsSchema = component.__pydantic_core_schema__ # pyright: ignore[reportAssignmentType] schema = core_schema while "schema" in schema: schema = schema["schema"] - model_schema: ModelFieldsSchema = schema # pyright:ignore[reportAssignmentType] + model_schema: ModelFieldsSchema = schema # pyright: ignore[reportAssignmentType] model_schema["fields"]["type"] = ModelField( type="model-field", schema=LiteralSchema( @@ -84,14 +84,14 @@ def gen_pipeline_schema() -> None: ), ) - PipelineComponents = Union[tuple(components)] # type: ignore[valid-type] + PipelineComponents = Union[tuple(components)] # pyright: ignore[reportInvalidTypeArguments,reportGeneralTypeIssues] AnnotatedPipelineComponents = Annotated[ PipelineComponents, Field(discriminator="type") ] class PipelineSchema(RootModel): root: Sequence[ - AnnotatedPipelineComponents # pyright:ignore[reportInvalidTypeForm] + AnnotatedPipelineComponents # pyright: ignore[reportInvalidTypeForm] ] print_schema(PipelineSchema) From 951cfdea8d67f255e2386292089e6bbb704ff05e Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 17:32:46 +0200 Subject: [PATCH 39/48] Fix `kpops --version` --- kpops/cli/main.py | 3 ++- kpops/const/__init__.py | 2 ++ pyproject.toml | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 kpops/const/__init__.py diff --git a/kpops/cli/main.py b/kpops/cli/main.py index d33712b3e..16d379456 100644 --- a/kpops/cli/main.py +++ b/kpops/cli/main.py @@ -12,6 +12,7 @@ collect_pipeline_paths, ) from kpops.config import ENV_PREFIX +from kpops.const import KPOPS, __version__ from kpops.utils.gen_schema import ( gen_config_schema, gen_defaults_schema, @@ -309,7 +310,7 @@ def clean( def version_callback(show_version: bool) -> None: if show_version: - # typer.echo(f"KPOps {__version__}") # FIXME: read version from package + typer.echo(f"{KPOPS} {__version__}") raise typer.Exit diff --git a/kpops/const/__init__.py b/kpops/const/__init__.py new file mode 100644 index 000000000..172f66927 --- /dev/null +++ b/kpops/const/__init__.py @@ -0,0 +1,2 @@ +__version__ = "6.0.1" +KPOPS = "KPOps" diff --git a/pyproject.toml b/pyproject.toml index 49ee0b459..23dd8ee3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,7 +63,7 @@ mkdocs-glightbox = "^0.3.1" mkdocs-exclude-search = "^0.6.5" mike = "^1.1.2" -[tool.poetry_bumpversion.file."kpops/__init__.py"] +[tool.poetry_bumpversion.file."kpops/const/__init__.py"] [tool.pyright] reportUnknownParameterType = "warning" From f87d71a7baa95b81b16eedac45ab8784d7470fd6 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Thu, 27 Jun 2024 19:49:59 +0200 Subject: [PATCH 40/48] Add test ensuring namespace compatibility --- tests/api/test_registry.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/api/test_registry.py b/tests/api/test_registry.py index df5223a71..ed496e0c8 100644 --- a/tests/api/test_registry.py +++ b/tests/api/test_registry.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib +from pathlib import Path from types import ModuleType import pytest @@ -36,6 +37,12 @@ class Unrelated: MODULE = SubComponent.__module__ +def test_namespace(): + """Ensure namespace package according to PEP 420.""" + assert not Path("kpops/__init__.py").exists() + assert not Path("kpops/components/__init__.py").exists() + + @pytest.mark.usefixtures("custom_components") def test_iter_namespace(): components_module = importlib.import_module("kpops.components") From 5ca958409ac67480c940964a1b9bd32b820ebfee Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 11:49:02 +0200 Subject: [PATCH 41/48] Update migration guide --- docs/docs/user/migration-guide/v6-v7.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/docs/user/migration-guide/v6-v7.md b/docs/docs/user/migration-guide/v6-v7.md index 74bf86c1e..78aaf0f31 100644 --- a/docs/docs/user/migration-guide/v6-v7.md +++ b/docs/docs/user/migration-guide/v6-v7.md @@ -1,5 +1,13 @@ # Migrate from V6 to V7 +## [Automatic loading of namespaced custom components](https://github.com/bakdata/kpops/pull/500) + +KPOps is now distributed as a Python namespace package (as defined by [PEP 420](https://peps.python.org/pep-0420/)). This allows us to standardize the namespace `kpops.components` for both builtin and custom pipeline components. + +Some imports need to be adjusted: + +**KPOps Python API** + ```diff -import kpops +import kpops.api as kpops From 1093c8d9e639695b2ef3cf907dd59ca8c642a682 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:38:54 +0200 Subject: [PATCH 42/48] Update migration guide --- docs/docs/user/migration-guide/v6-v7.md | 50 ++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/docs/docs/user/migration-guide/v6-v7.md b/docs/docs/user/migration-guide/v6-v7.md index 78aaf0f31..53c83c796 100644 --- a/docs/docs/user/migration-guide/v6-v7.md +++ b/docs/docs/user/migration-guide/v6-v7.md @@ -4,7 +4,7 @@ KPOps is now distributed as a Python namespace package (as defined by [PEP 420](https://peps.python.org/pep-0420/)). This allows us to standardize the namespace `kpops.components` for both builtin and custom pipeline components. -Some imports need to be adjusted: +As a result of the restructure, some imports need to be adjusted: **KPOps Python API** @@ -13,4 +13,50 @@ Some imports need to be adjusted: +import kpops.api as kpops ``` -TODO +**builtin KPOps components** + +```diff +-from kpops.components import ( +- HelmApp, +- KafkaApp, +- KafkaConnector, +- KafkaSinkConnector, +- KafkaSourceConnector, +- KubernetesApp, +- StreamsBootstrap, +- ProducerApp, +- StreamsApp, +- PipelineComponent, +- StreamsApp, +- ProducerApp, +-) ++from kpops.components.base_components import ( ++ HelmApp, ++ KafkaApp, ++ KafkaConnector, ++ KafkaSinkConnector, ++ KafkaSourceConnector, ++ KubernetesApp, ++ PipelineComponent, ++) ++from kpops.components.streams_bootstrap import ( ++ StreamsBootstrap, ++ StreamsApp, ++ ProducerApp, ++) +``` + +### your custom KPOps components + +#### config.yaml + +```diff +-components_module: components +``` + +#### Python module + +```diff +-components/__init__.py ++kpops/components/custom/__init__.py +``` From b329f60b1efc2fc22a1284e2894fd9796150608b Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:40:30 +0200 Subject: [PATCH 43/48] Update config --- config.yaml | 1 - docs/docs/resources/pipeline-config/config.yaml | 6 ++---- tests/cli/resources/config.yaml | 1 - tests/cli/resources/empty_module/config.yaml | 1 - 4 files changed, 2 insertions(+), 7 deletions(-) diff --git a/config.yaml b/config.yaml index 7d0e97a54..359b51a21 100644 --- a/config.yaml +++ b/config.yaml @@ -1,3 +1,2 @@ kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092" -components_module: tests.pipeline.test_components pipeline_base_dir: tests/pipeline diff --git a/docs/docs/resources/pipeline-config/config.yaml b/docs/docs/resources/pipeline-config/config.yaml index 862a49ac0..9a7d53578 100644 --- a/docs/docs/resources/pipeline-config/config.yaml +++ b/docs/docs/resources/pipeline-config/config.yaml @@ -1,14 +1,12 @@ # CONFIGURATION # -# Custom Python module defining project-specific KPOps components -components_module: null # Base directory to the pipelines (default is current working directory) pipeline_base_dir: . # The Kafka brokers address. # REQUIRED kafka_brokers: "http://broker1:9092,http://broker2:9092" # Configure the topic name variables you can use in the pipeline definition. -topic_name_config: +topic_name_config: # Configures the value for the variable ${output_topic_name} default_output_topic_name: ${pipeline.name}-${component.name} # Configures the value for the variable ${error_topic_name} @@ -42,7 +40,7 @@ helm_config: # Kubernetes API version used for Capabilities.APIVersions api_version: null # Configure Helm Diff. -helm_diff_config: +helm_diff_config: # Set of keys that should not be checked. ignore: - name diff --git a/tests/cli/resources/config.yaml b/tests/cli/resources/config.yaml index 046c98d2a..79261856b 100644 --- a/tests/cli/resources/config.yaml +++ b/tests/cli/resources/config.yaml @@ -1,2 +1 @@ kafka_brokers: http://127.0.0.1:9092 -components_module: tests.cli.test_schema_generation diff --git a/tests/cli/resources/empty_module/config.yaml b/tests/cli/resources/empty_module/config.yaml index 735b3904a..79261856b 100644 --- a/tests/cli/resources/empty_module/config.yaml +++ b/tests/cli/resources/empty_module/config.yaml @@ -1,2 +1 @@ kafka_brokers: http://127.0.0.1:9092 -components_module: tests.cli.resources.empty_module From 84dd087996c6a933804a14c216a1426a37d95a31 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:41:14 +0200 Subject: [PATCH 44/48] Update config --- docs/docs/resources/pipeline-config/config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/docs/resources/pipeline-config/config.yaml b/docs/docs/resources/pipeline-config/config.yaml index 9a7d53578..d8c5433b7 100644 --- a/docs/docs/resources/pipeline-config/config.yaml +++ b/docs/docs/resources/pipeline-config/config.yaml @@ -25,9 +25,6 @@ kafka_rest: kafka_connect: # Address of Kafka Connect. url: "http://localhost:8083" -# The timeout in seconds that specifies when actions like deletion or deploy -# timeout. -timeout: 300 # Flag for `helm upgrade --install`. # Create the release namespace if not present. create_namespace: false From 31636cadf4e518b219dc6bc0a12ad3cc17176847 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:42:32 +0200 Subject: [PATCH 45/48] Remove unnecessary test resources --- tests/cli/resources/empty_module/__init__.py | 0 tests/cli/resources/empty_module/config.yaml | 1 - tests/cli/resources/no_module/config.yaml | 1 - 3 files changed, 2 deletions(-) delete mode 100644 tests/cli/resources/empty_module/__init__.py delete mode 100644 tests/cli/resources/empty_module/config.yaml delete mode 100644 tests/cli/resources/no_module/config.yaml diff --git a/tests/cli/resources/empty_module/__init__.py b/tests/cli/resources/empty_module/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/tests/cli/resources/empty_module/config.yaml b/tests/cli/resources/empty_module/config.yaml deleted file mode 100644 index 79261856b..000000000 --- a/tests/cli/resources/empty_module/config.yaml +++ /dev/null @@ -1 +0,0 @@ -kafka_brokers: http://127.0.0.1:9092 diff --git a/tests/cli/resources/no_module/config.yaml b/tests/cli/resources/no_module/config.yaml deleted file mode 100644 index 79261856b..000000000 --- a/tests/cli/resources/no_module/config.yaml +++ /dev/null @@ -1 +0,0 @@ -kafka_brokers: http://127.0.0.1:9092 From 475f2cee2423e52ec7a2bf0bc1b1288114ed801f Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Wed, 3 Jul 2024 13:47:25 +0200 Subject: [PATCH 46/48] Refactor module name --- kpops/api/registry.py | 2 +- kpops/const/__init__.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 90c0e81a9..3a9af7412 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -13,11 +13,11 @@ from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent +from kpops.const import KPOPS_MODULE if TYPE_CHECKING: from collections.abc import Iterator -KPOPS_MODULE = "kpops." T = TypeVar("T") ClassDict = dict[str, type[T]] # type -> class diff --git a/kpops/const/__init__.py b/kpops/const/__init__.py index 172f66927..c55d7127f 100644 --- a/kpops/const/__init__.py +++ b/kpops/const/__init__.py @@ -1,2 +1,3 @@ __version__ = "6.0.1" KPOPS = "KPOps" +KPOPS_MODULE = "kpops." From 56a5ce94f50035e5098e544ad51ebf6c10005372 Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 8 Jul 2024 10:57:12 +0200 Subject: [PATCH 47/48] Refactor debug log --- kpops/api/registry.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 3a9af7412..7cf5ba216 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -11,6 +11,8 @@ from types import ModuleType from typing import TYPE_CHECKING, TypeVar +import typer + from kpops.api.exception import ClassNotFoundError from kpops.components.base_components.pipeline_component import PipelineComponent from kpops.const import KPOPS_MODULE @@ -69,14 +71,10 @@ def find_class(modules: Iterable[ModuleType], base: type[T]) -> type[T]: def import_module(module_name: str) -> ModuleType: module = importlib.import_module(module_name) - # TODO: remove? unnecessary now - if module.__file__ and not module_name.startswith(KPOPS_MODULE): - file_path = Path(module.__file__) - try: - rel_path = file_path.relative_to(Path.cwd()) - log.debug(f"Picked up: {rel_path}") - except ValueError: - log.debug(f"Picked up: {file_path}") + if module.__file__: + log.debug( + f"Loading {typer.style(module.__name__,bold=True)} ({module.__file__})" + ) return module From e1f3edc4d2c2269042be14992c5b346c34e509cb Mon Sep 17 00:00:00 2001 From: Salomon Popp Date: Mon, 8 Jul 2024 11:01:14 +0200 Subject: [PATCH 48/48] Rename TypeVar for plugins --- kpops/api/registry.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kpops/api/registry.py b/kpops/api/registry.py index 7cf5ba216..37c8e5f9c 100644 --- a/kpops/api/registry.py +++ b/kpops/api/registry.py @@ -21,8 +21,8 @@ from collections.abc import Iterator -T = TypeVar("T") -ClassDict = dict[str, type[T]] # type -> class +_PluginT = TypeVar("_PluginT") +ClassDict = dict[str, type[_PluginT]] # type -> class sys.path.append(str(Path.cwd())) log = logging.getLogger("Registry") @@ -62,7 +62,7 @@ def iter_component_modules() -> Iterator[ModuleType]: yield from _iter_namespace(kpops.components) -def find_class(modules: Iterable[ModuleType], base: type[T]) -> type[T]: +def find_class(modules: Iterable[ModuleType], base: type[_PluginT]) -> type[_PluginT]: try: return next(_find_classes(modules, base=base)) except StopIteration as e: @@ -78,7 +78,9 @@ def import_module(module_name: str) -> ModuleType: return module -def _find_classes(modules: Iterable[ModuleType], base: type[T]) -> Iterator[type[T]]: +def _find_classes( + modules: Iterable[ModuleType], base: type[_PluginT] +) -> Iterator[type[_PluginT]]: for module in modules: for _, _class in inspect.getmembers(module, inspect.isclass): if not __filter_internal_kpops_classes(