Merge pull request #558 from bakdata/v9

KPOps V9
bakdata · Dec 20, 2024 · febf664 · febf664
2 parents 337b231 + 21ce117
commit febf664
Show file tree

Hide file tree

Showing 90 changed files with 3,252 additions and 548 deletions.
diff --git a/.github/actions/update-docs/action.yml b/.github/actions/update-docs/action.yml
@@ -27,8 +27,8 @@ runs:
     - name: Install Python and set up Poetry
       uses: bakdata/ci-templates/actions/[email protected]
       with:
+        python-version: "3.11"
         poetry-version: "1.5.1"
-        python-version: "3.10"
 
     - name: Install docs dependencies
       shell: bash

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -21,7 +21,7 @@ jobs:
         os:
           - ubuntu-22.04
           - windows-2022
-        python-version: ["3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -32,8 +32,8 @@ jobs:
       - name: Install Python and set up Poetry
         uses: bakdata/ci-templates/actions/[email protected]
         with:
-          poetry-version: "1.7.1"
           python-version: ${{ matrix.python-version }}
+          poetry-version: "1.7.1"
 
       - name: Check Poetry lock file consistency
         run: poetry lock --check
@@ -43,7 +43,7 @@ jobs:
 
       - name: Lint (ruff)
         run: |
-          if [[ "$RUNNER_OS" == "Linux" && "${{ matrix.python-version }}" == "3.10" ]]
+          if [[ "$RUNNER_OS" == "Linux" && "${{ matrix.python-version }}" == "3.11" ]]
           then
             poetry run ruff check . --config pyproject.toml --output-format=github --no-fix
           else
@@ -55,7 +55,7 @@ jobs:
 
       - name: Typing (pyright)
         run: |
-          if [[ "$RUNNER_OS" == "Linux" && "${{ matrix.python-version }}" == "3.10" ]]
+          if [[ "$RUNNER_OS" == "Linux" && "${{ matrix.python-version }}" == "3.11" ]]
           then
             echo "::add-matcher::.github/pyright-matcher.json"
             poetry run pre-commit run pyright --all-files
@@ -93,6 +93,7 @@ jobs:
     needs: [test]
     uses: bakdata/ci-templates/.github/workflows/[email protected]
     with:
+      python-version: "3.11"
       poetry-version: "1.7.1"
     secrets:
       pypi-token: ${{ secrets.TEST_PYPI_TOKEN }}

diff --git a/.github/workflows/publish.yaml b/.github/workflows/publish.yaml
@@ -9,6 +9,7 @@ jobs:
     uses: bakdata/ci-templates/.github/workflows/[email protected]
     with:
       publish-to-test: false
+      python-version: "3.11"
       poetry-version: "1.7.1"
     secrets:
       pypi-token: "${{ secrets.PYPI_TOKEN }}"
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -19,6 +19,7 @@ jobs:
     name: Release
     with:
       release-type: ${{ inputs.release-type }}
+      python-version: "3.11"
       poetry-version: "1.7.1"
       changelog: true
       changelog-config: "./.github/changelog-config.json"

diff --git a/config.yaml b/config.yaml
@@ -1,2 +1,5 @@
 kafka_brokers: "http://k8kafka-cp-kafka-headless.kpops.svc.cluster.local:9092"
 pipeline_base_dir: tests/pipeline
+strimzi_topic:
+  label:
+     strimzi.io/cluster: my-cluster
diff --git a/docs/docs/resources/variables/cli_env_vars.env b/docs/docs/resources/variables/cli_env_vars.env
@@ -14,6 +14,8 @@ KPOPS_DOTENV_PATH # No default value, not required
 # Suffix your environment files with this value (e.g.
 # defaults_development.yaml for environment=development).
 KPOPS_ENVIRONMENT # No default value, not required
+# How KPOps should operate.
+KPOPS_OPERATION_MODE=managed
 # Paths to dir containing 'pipeline.yaml' or files named
 # 'pipeline.yaml'.
 KPOPS_PIPELINE_PATHS # No default value, required

diff --git a/docs/docs/resources/variables/cli_env_vars.md b/docs/docs/resources/variables/cli_env_vars.md
@@ -5,5 +5,6 @@ These variables take precedence over the commands' flags. If a variable is set,
 |KPOPS_CONFIG_PATH   |.            |False   |Path to the dir containing config.yaml files                                                                                                                                |
 |KPOPS_DOTENV_PATH   |             |False   |Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.                                         |
 |KPOPS_ENVIRONMENT   |             |False   |The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).|
+|KPOPS_OPERATION_MODE|managed      |False   |How KPOps should operate.                                                                                                                                                   |
 |KPOPS_PIPELINE_PATHS|             |True    |Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.                                                                                                     |
 |KPOPS_PIPELINE_STEPS|             |False   |Comma separated list of steps to apply the command on                                                                                                                       |
diff --git a/docs/docs/resources/variables/config_env_vars.env b/docs/docs/resources/variables/config_env_vars.env
@@ -58,3 +58,9 @@ KPOPS_HELM_DIFF_CONFIG__IGNORE # No default value, required
 # Whether to retain clean up jobs in the cluster or uninstall the,
 # after completion.
 KPOPS_RETAIN_CLEAN_JOBS=False
+# strimzi_topic
+# Configuration for Strimzi Kafka Topics.
+KPOPS_STRIMZI_TOPIC # No default value, not required
+# operation_mode
+# The operation mode of KPOps (managed, manifest, argo).
+KPOPS_OPERATION_MODE=managed
diff --git a/docs/docs/resources/variables/config_env_vars.md b/docs/docs/resources/variables/config_env_vars.md
@@ -19,3 +19,5 @@ These variables take precedence over the settings in `config.yaml`. Variables ma
 |KPOPS_HELM_CONFIG__API_VERSION                    |                                        |False   |Kubernetes API version used for `Capabilities.APIVersions`                        |helm_config.api_version                    |
 |KPOPS_HELM_DIFF_CONFIG__IGNORE                    |                                        |True    |Set of keys that should not be checked.                                           |helm_diff_config.ignore                    |
 |KPOPS_RETAIN_CLEAN_JOBS                           |False                                   |False   |Whether to retain clean up jobs in the cluster or uninstall the, after completion.|retain_clean_jobs                          |
+|KPOPS_STRIMZI_TOPIC                               |                                        |False   |Configuration for Strimzi Kafka Topics.                                           |strimzi_topic                              |
+|KPOPS_OPERATION_MODE                              |managed                                 |False   |The operation mode of KPOps (managed, manifest, argo).                            |operation_mode                             |
diff --git a/docs/docs/schema/config.json b/docs/docs/schema/config.json
@@ -153,6 +153,25 @@
             "title": "SchemaRegistryConfig",
             "type": "object"
         },
+        "StrimziTopicConfig": {
+            "additionalProperties": false,
+            "description": "Configuration for Strimzi Kafka Topics.",
+            "properties": {
+                "label": {
+                    "additionalProperties": {
+                        "type": "string"
+                    },
+                    "description": "The label to identify the KafkaTopic resources managed by the Topic Operator. This does not have to be the name of the Kafka cluster. It can be the label assigned to the KafkaTopic resource. If you deploy more than one Topic Operator, the labels must be unique for each. That is, the operators cannot manage the same resources.",
+                    "title": "Label",
+                    "type": "object"
+                }
+            },
+            "required": [
+                "label"
+            ],
+            "title": "StrimziTopicConfig",
+            "type": "object"
+        },
         "TopicNameConfig": {
             "additionalProperties": false,
             "description": "Configure the topic name variables you can use in the pipeline definition.",
@@ -265,6 +284,18 @@
             },
             "description": "Configuration for Schema Registry."
         },
+        "strimzi_topic": {
+            "anyOf": [
+                {
+                    "$ref": "#/$defs/StrimziTopicConfig"
+                },
+                {
+                    "type": "null"
+                }
+            ],
+            "default": null,
+            "description": "Configuration for Strimzi Kafka Topics."
+        },
         "topic_name_config": {
             "allOf": [
                 {

diff --git a/docs/docs/user/core-concepts/operation-mode.md b/docs/docs/user/core-concepts/operation-mode.md
@@ -0,0 +1,119 @@
+# Operation Modes in KPOps
+
+KPOps supports three operation modes—`managed`, `manifest`, and `argo`. These modes determine how resources are managed and allow users to tailor their deployment strategy.
+
+- **Managed Mode** (default): KPOps uses Helm, and communicates with services like Kafka Rest Proxy, and Kafka Connect under the hood to manage the installation/(graceful) deletion of applications, creation/deletion of Kafka topics, creation/deletion of Connectors defined in your `pipeline.yaml`.
+- **Manifest Mode**: Focuses on generating Kubernetes manifests.
+- **Argo Mode**: Extends the functionality to include ArgoCD-specific hooks for certain operations, facilitating GitOps workflows with automated cleanup and reset tasks.
+
+---
+
+## Configuring Operation Modes
+
+You can configure the operation mode using one of the following methods:
+
+1. **Command-Line Option**: Pass the `--operation-mode <OPERATION>` flag when running a CLI command. Refer to the [CLI commands documentation](https://bakdata.github.io/kpops/9.0/user/references/cli-commands/#kpops-deploy) for more details.
+
+2. **Environment Variable**: Set the operation mode by defining the `KPOPS_OPERATION_MODE` environment variable.
+
+---
+
+## Generated Resources by Mode and Operation
+
+### `deploy`
+
+#### **Manifest Mode**
+
+- **streams-bootstrap Applications**:
+  - Depending on your pipeline configuration, Kubernetes `Job`, `Deployment`, `ConfigMap`, and `Service` resources.
+  - Please refer to [streams-bootstrap Helm Charts](https://github.com/bakdata/streams-bootstrap/tree/master/charts).
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+
+#### **Argo Mode**
+
+- **streams-bootstrap Applications**:
+  - Depending on your pipeline configuration, Kubernetes `Job`, `Deployment`, `ConfigMap`, and `Service` resources.
+  - Additional Argo `sync-wave` annotation to ensure Kafka topics are created first (default `sync-wave=0`) before deploying apps (lower priority `sync-wave>0`). All components of each sync wave are deployed in parallel by Argo.
+  - Please refer to [streams-bootstrap Helm Charts](https://github.com/bakdata/streams-bootstrap/tree/master/charts).
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+- **Cleanup Jobs**:
+  - Kubernetes `Job` resources configured **with** ArgoCD `PostDelete` hooks, ensuring cleanup tasks are executed after ArgoCD application deletion.
+
+---
+
+### `reset`
+
+#### **Manifest Mode**
+
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+- **Reset Jobs**:
+  - Kubernetes `Job` resources for resetting Kafka Streams application states.
+
+#### **Argo Mode**
+
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+- **Reset Jobs**:
+  - Kubernetes `Job` resources **without** ArgoCD `PostDelete` hooks, providing a simpler reset process.
+
+---
+
+### `clean`
+
+#### **Manifest Mode**
+
+- **Clean Jobs**:
+  - Kubernetes `Job` resources for cleaning up temporary resources or artifacts using application container images.
+
+#### **Argo Mode**
+
+- **Not Applicable**:
+  - The `clean` command is not supported in Argo mode. The clean is instead achieved through cleanup job hooks during the `deploy` command.
+
+---
+
+### `destroy`
+
+#### **Manifest Mode**
+
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+
+#### **Argo Mode**
+
+- **Topics**:
+  - Strimzi `KafkaTopic` CRDs.
+
+---
+
+## Use Cases for Each Mode
+
+### Manifest Mode
+
+- **Flexibility**: Use the generated manifests in manual workflows or integrate with any Kubernetes deployment tool.
+- **Version Control**: Commit generated manifests to a Git repository for tracking changes and rollback.
+
+### Argo Mode
+
+- **GitOps Integration**: Simplifies workflows when using ArgoCD for automated deployments and lifecycle management.
+- **PostDelete Hooks**: Automatically cleans up resources after deletion of ArgoCD applications.
+
+---
+
+## Summary of Resource Generation by Operation and Mode
+
+| Resource Type | `deploy`                         | `reset`             | `clean`             | `destroy`           |
+| ------------- | -------------------------------- | ------------------- | ------------------- | ------------------- |
+| Producer Apps | Manifest: Generated              | N/A                 | N/A                 | N/A                 |
+|               | Argo: Generated                  |                     |                     |                     |
+| Streams Apps  | Manifest: Generated              | N/A                 | N/A                 | N/A                 |
+|               | Argo: Generated                  |                     |                     |                     |
+| Topics        | Manifest: Generated              | Manifest: Generated | N/A                 | Manifest: Generated |
+|               | Argo: Generated                  | Argo: Generated     |                     | Argo: Generated     |
+| Cleanup Jobs  | Manifest: N/A                    | N/A                 | Manifest: Generated | N/A                 |
+|               | Argo: With `PostDelete` hooks    | N/A                 | N/A                 | N/A                 |
+| Reset Jobs    | Manifest: N/A                    | Manifest: Generated | N/A                 | N/A                 |
+|               | Argo: Without `PostDelete` hooks |                     |                     |                     |
diff --git a/docs/docs/user/migration-guide/v8-v9.md b/docs/docs/user/migration-guide/v8-v9.md
@@ -0,0 +1,42 @@
+# Migrate from V8 to V9
+
+## [Introduce KPOps operation and manifest resources for deployment](https://github.com/bakdata/kpops/pull/541)
+
+The `kpops manifest` command and `kpops.manifest()` API have been **removed**.
+
+Resource manifesting is now integrated into the _operation_ commands (`deploy`, `destroy`, `reset`, `clean`) through the new **operation mode** feature.
+
+To manifest resources, you can:
+
+- Pass `--operation-mode manifest` when executing `kpops` commands.
+- Set the operation mode by defining the `KPOPS_OPERATION_MODE` environment variable.
+
+## [Manifest toSection with Strimzi KafkaTopic](https://github.com/bakdata/kpops/pull/545)
+
+KPOps now supports generating valid Kubernetes KafkaTopic resources compatible with [Strimzi](https://github.com/strimzi/strimzi-kafka-operator/blob/main/examples/topic/kafka-topic.yaml). When using manifest or argo as the operation_mode, you must specify the Strimzi cluster label to ensure the topics are recognized by the deployed Strimzi Topic Operator.
+
+```diff
+operation_mode: manifest
+
++ strimzi_topic:
++  label:
++     strimzi.io/cluster: my-cluster
+
+# rest of your config
+```
+
+<!-- dprint-ignore-start -->
+
+!!! info Standalone topic operator deployment
+     Refer to the [Strimzi documentation on deploying a standalone topic operator](https://strimzi.io/docs/operators/latest/deploying#deploying-the-topic-operator-standalone-str) for more details.
+
+<!-- dprint-ignore-end -->
+
+## [Drop support for Python 3.10](https://github.com/bakdata/kpops/pull/561)
+
+KPOps V9 no longer supports Python 3.10. Ensure your environment is running Python 3.11 to 3.12.
+
+#### Action Required:
+
+Upgrade your Python version to a supported version (3.11 or 3.12).
+Update your virtual environments and CI pipelines to reflect this change.
diff --git a/docs/docs/user/references/cli-commands.md b/docs/docs/user/references/cli-commands.md
@@ -20,7 +20,6 @@ $ kpops [OPTIONS] COMMAND [ARGS]...
 * `destroy`: Destroy pipeline steps
 * `generate`: Generate enriched pipeline representation
 * `init`: Initialize a new KPOps project.
-* `manifest`: Render final resource representation
 * `reset`: Reset pipeline steps
 * `schema`: Generate JSON schema.
 
@@ -48,6 +47,7 @@ $ kpops clean [OPTIONS] PIPELINE_PATHS...
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
 * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially.  [default: no-parallel]
+* `--operation-mode [argo|manifest|managed]`: How KPOps should operate.  [env var: KPOPS_OPERATION_MODE; default: managed]
 * `--help`: Show this message and exit.
 
 ## `kpops deploy`
@@ -74,6 +74,7 @@ $ kpops deploy [OPTIONS] PIPELINE_PATHS...
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
 * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially.  [default: no-parallel]
+* `--operation-mode [argo|manifest|managed]`: How KPOps should operate.  [env var: KPOPS_OPERATION_MODE; default: managed]
 * `--help`: Show this message and exit.
 
 ## `kpops destroy`
@@ -100,6 +101,7 @@ $ kpops destroy [OPTIONS] PIPELINE_PATHS...
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
 * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially.  [default: no-parallel]
+* `--operation-mode [argo|manifest|managed]`: How KPOps should operate.  [env var: KPOPS_OPERATION_MODE; default: managed]
 * `--help`: Show this message and exit.
 
 ## `kpops generate`
@@ -142,31 +144,7 @@ $ kpops init [OPTIONS] PATH
 
 **Options**:
 
-* `--config-include-opt / --no-config-include-opt`: Whether to include non-required settings in the generated 'config.yaml'  [default: no-config-include-opt]
-* `--help`: Show this message and exit.
-
-## `kpops manifest`
-
-In addition to generate, render final resource representation for each pipeline step, e.g. Kubernetes manifests.
-
-**Usage**:
-
-```console
-$ kpops manifest [OPTIONS] PIPELINE_PATHS...
-```
-
-**Arguments**:
-
-* `PIPELINE_PATHS...`: Paths to dir containing 'pipeline.yaml' or files named 'pipeline.yaml'.  [env var: KPOPS_PIPELINE_PATHS;required]
-
-**Options**:
-
-* `--dotenv FILE`: Path to dotenv file. Multiple files can be provided. The files will be loaded in order, with each file overriding the previous one.  [env var: KPOPS_DOTENV_PATH]
-* `--config DIRECTORY`: Path to the dir containing config.yaml files  [env var: KPOPS_CONFIG_PATH; default: .]
-* `--steps TEXT`: Comma separated list of steps to apply the command on  [env var: KPOPS_PIPELINE_STEPS]
-* `--filter-type [include|exclude]`: Whether the --steps option should include/exclude the steps  [default: include]
-* `--environment TEXT`: The environment you want to generate and deploy the pipeline to. Suffix your environment files with this value (e.g. defaults_development.yaml for environment=development).   [env var: KPOPS_ENVIRONMENT]
-* `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
+* `--config-include-optional / --no-config-include-optional`: Whether to include non-required settings in the generated 'config.yaml'  [default: no-config-include-optional]
 * `--help`: Show this message and exit.
 
 ## `kpops reset`
@@ -193,6 +171,7 @@ $ kpops reset [OPTIONS] PIPELINE_PATHS...
 * `--dry-run / --execute`: Whether to dry run the command or execute it  [default: dry-run]
 * `--verbose / --no-verbose`: Enable verbose printing  [default: no-verbose]
 * `--parallel / --no-parallel`: Enable or disable parallel execution of pipeline steps. If enabled, multiple steps can be processed concurrently. If disabled, steps will be processed sequentially.  [default: no-parallel]
+* `--operation-mode [argo|manifest|managed]`: How KPOps should operate.  [env var: KPOPS_OPERATION_MODE; default: managed]
 * `--help`: Show this message and exit.
 
 ## `kpops schema`

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -131,6 +131,7 @@ nav:
               - Migrate from v5 to v6: user/migration-guide/v5-v6.md
               - Migrate from v6 to v7: user/migration-guide/v6-v7.md
               - Migrate from v7 to v8: user/migration-guide/v7-v8.md
+              - Migrate from v8 to v9: user/migration-guide/v8-v9.md
           - CLI usage: user/references/cli-commands.md
           - Editor integration: user/references/editor-integration.md
           - CI integration: