From 240d0eb979e04c9106fc2b743683e6173e6f95ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 17 Dec 2024 14:57:11 +0100 Subject: [PATCH] Remove demo pipeline from values file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Høydahl --- charts/data-prepper/CHANGELOG.md | 6 +- charts/data-prepper/Chart.yaml | 2 +- charts/data-prepper/README.md | 13 +- charts/data-prepper/templates/NOTES.txt | 8 + .../data-prepper/templates/demoPipeline.yaml | 25 +++ charts/data-prepper/templates/deployment.yaml | 6 +- charts/data-prepper/values.yaml | 168 ++---------------- 7 files changed, 70 insertions(+), 158 deletions(-) create mode 100644 charts/data-prepper/templates/demoPipeline.yaml diff --git a/charts/data-prepper/CHANGELOG.md b/charts/data-prepper/CHANGELOG.md index d0b61260..d0d66ee9 100644 --- a/charts/data-prepper/CHANGELOG.md +++ b/charts/data-prepper/CHANGELOG.md @@ -4,7 +4,11 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.2.0] +### Added +- Demo pipeline not part of values.yaml. New value `pipelineConfig.demoPipeline` to enable demo pipeline. + +## [0.1.0] ### Added - Create initial version of data-prepper helm chart diff --git a/charts/data-prepper/Chart.yaml b/charts/data-prepper/Chart.yaml index dd53bd76..dfeeea98 100644 --- a/charts/data-prepper/Chart.yaml +++ b/charts/data-prepper/Chart.yaml @@ -15,7 +15,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 +version: 0.2.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/charts/data-prepper/README.md b/charts/data-prepper/README.md index d35c92f7..8987a65b 100644 --- a/charts/data-prepper/README.md +++ b/charts/data-prepper/README.md @@ -27,10 +27,10 @@ helm repo add opensearch https://opensearch-project.github.io/helm-charts/ helm repo update ``` -* Install the Data Prepper chart with: +* Install the Data Prepper chart with demo pipeline: ```bash -helm install my-data-prepper-release opensearch/data-prepper +helm install my-data-prepper-release opensearch/data-prepper --set pipelineConfig.demoPipeline=true ``` Replace my-data-prepper-release with your desired release name. @@ -38,7 +38,7 @@ Replace my-data-prepper-release with your desired release name. ## Configuration The Data Prepper Helm chart comes with a variety of configuration options to tailor the deployment to your needs. -The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own values.yaml file during installation or by specifying configuration options with --set flags. +The default values are specified in the [values.yaml](values.yaml) file. You can override these values by providing your own `values.yaml` file during installation or by specifying configuration options with --set flags. For a detailed list of configuration options, refer to the values.yaml file or the [Data Prepper documentation](https://opensearch.org/docs/latest/data-prepper/managing-data-prepper/configuring-data-prepper/). @@ -98,8 +98,11 @@ We welcome contributions! Please read our [CONTRIBUTING.md](../../CONTRIBUTING.m | ingress.tls | list | `[]` | | | nameOverride | string | `""` | Override the default name for the deployment | | nodeSelector | object | `{}` | | -| pipelineConfig | object | `{"config":{"simple-sample-pipeline":{"buffer":{"bounded_blocking":{"batch_size":256,"buffer_size":1024}},"delay":5000,"processor":[{"string_converter":{"upper_case":true}}],"sink":[{"stdout":null}],"source":{"random":null},"workers":2}},"enabled":true,"existingSecret":""}` | Pipeline configuration | -| pipelineConfig.existingSecret | string | `""` | The name of the existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | +| pipelineConfig | object | (See below) | Pipeline configuration | +| pipelineConfig.enabled | boolean | `false` | Enable inline configuration in `config` sub key. | +| pipelineConfig.config | object | `{}` | Pipeline configuration file inline if `enabled` is set to true | +| pipelineConfig.demoPipeline | boolean | `false` | If set, a demo pipeline will be provisioned with source `random` and sink `stdout`. | +| pipelineConfig.existingSecret | string | `""` | The name of an existing secret containing the pipeline configuration. If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. | | podAnnotations | object | `{}` | | | podLabels | object | `{}` | | | podSecurityContext | object | `{}` | | diff --git a/charts/data-prepper/templates/NOTES.txt b/charts/data-prepper/templates/NOTES.txt index cae9ebaf..73f9fe18 100644 --- a/charts/data-prepper/templates/NOTES.txt +++ b/charts/data-prepper/templates/NOTES.txt @@ -20,3 +20,11 @@ echo "Visit http://127.0.0.1:8080 to use your application" kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT {{- end }} + +{{ if .Values.pipelineConfig.demoPipeline }} +The demo pipeline configuration is enabled, using the `random` source and `stdout` sink. +{{- else if .Values.pipelineConfig.enabled }} +Inline pipeline configuration is enabled. Please refer to the values.yaml file for the configuration. +{{- else }} +Pipeline configuration from secret {{ .Values.pipelineConfig.existingSecret }} is enabled. +{{- end }} \ No newline at end of file diff --git a/charts/data-prepper/templates/demoPipeline.yaml b/charts/data-prepper/templates/demoPipeline.yaml new file mode 100644 index 00000000..8820516f --- /dev/null +++ b/charts/data-prepper/templates/demoPipeline.yaml @@ -0,0 +1,25 @@ +{{- if .Values.pipelineConfig.demoPipeline }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "data-prepper.fullname" . }}-demo-pipeline + labels: + {{- include "data-prepper.labels" . | nindent 4 }} +type: Opaque +stringData: + pipelines.yaml: | + simple-sample-pipeline: + workers: 2 # the number of workers + delay: 5000 # in milliseconds, how long workers wait between read attempts + source: + random: {} + buffer: + bounded_blocking: + buffer_size: 1024 # max number of records the buffer accepts + batch_size: 256 # max number of records the buffer drains after each read + processor: + - string_converter: + upper_case: true + sink: + - stdout: {} +{{- end }} diff --git a/charts/data-prepper/templates/deployment.yaml b/charts/data-prepper/templates/deployment.yaml index 3db562c7..e83d971f 100644 --- a/charts/data-prepper/templates/deployment.yaml +++ b/charts/data-prepper/templates/deployment.yaml @@ -90,10 +90,12 @@ spec: name: {{ include "data-prepper.fullname" . }}-config - name: data-prepper-pipelines secret: - {{- if .Values.pipelineConfig.enabled }} + {{- if .Values.pipelineConfig.demoPipeline }} + secretName: {{ include "data-prepper.fullname" . }}-demo-pipeline + {{- else if .Values.pipelineConfig.enabled }} secretName: {{ include "data-prepper.fullname" . }}-pipeline {{- else }} - secretName: {{ required "A valid .Values.pipelineConfig.existingSecret entry required!" .Values.pipelineConfig.existingSecret }} + secretName: {{ required "A valid pipeline configuration is required! Please set one of:\npipelineConfig.existingSecret - for your custom pipeline secret\npipelineConfig.enabled - for inline configuration in values.yaml or\npipelineConfig.demoPipeline=true - for a sample demo pipeline using random source and stdout sink." .Values.pipelineConfig.existingSecret }} {{- end }} {{- with .Values.volumes }} {{- toYaml . | nindent 8 }} diff --git a/charts/data-prepper/values.yaml b/charts/data-prepper/values.yaml index b63cf837..cb16dc31 100644 --- a/charts/data-prepper/values.yaml +++ b/charts/data-prepper/values.yaml @@ -97,160 +97,30 @@ config: # -- Pipeline configuration pipelineConfig: - # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. - enabled: true + # If enabled, a secret containing a demo pipeline configuration with random source and stdout sink will be created. + demoPipeline: false # -- The name of the existing secret containing the pipeline configuration. # If enabled is false existingSecret is used. The existingSecret must have a key named `pipelines.yaml`. existingSecret: "" + # If enabled, a secret containing the pipeline configuration will be created based on the 'config' section below. + enabled: false # The configuration of the pipeline see https://opensearch.org/docs/2.4/data-prepper/pipelines/pipelines/ config: - ## Simple Example - simple-sample-pipeline: - workers: 2 # the number of workers - delay: 5000 # in milliseconds, how long workers wait between read attempts - source: - random: - buffer: - bounded_blocking: - buffer_size: 1024 # max number of records the buffer accepts - batch_size: 256 # max number of records the buffer drains after each read - processor: - - string_converter: - upper_case: true - sink: - - stdout: - - ## More Complex example - # otel-logs-pipeline: - # workers: 5 - # delay: 10 - # source: - # otel_logs_source: - # ssl: false - # buffer: - # bounded_blocking: - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: events-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-trace-pipeline: - # # workers is the number of threads processing data in each pipeline. - # # We recommend same value for all pipelines. - # # default value is 1, set a value based on the machine you are running Data Prepper - # workers: 8 - # # delay in milliseconds is how often the worker threads should process data. - # # Recommend not to change this config as we want the otel-trace-pipeline to process as quick as possible - # # default value is 3_000 ms - # delay: "100" - # source: - # otel_trace_source: - # ssl: false # Change this to enable encryption in transit - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - pipeline: - # name: "raw-traces-pipeline" - # - pipeline: - # name: "otel-service-map-pipeline" - # raw-traces-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # buffer: - # bounded_blocking: - # buffer_size: 25600 # max number of records the buffer accepts - # batch_size: 400 # max number of records the buffer drains after each read - # processor: - # - otel_traces: - # - otel_trace_group: - # hosts: [ "https://opensearch-cluster-master:9200" ] - # insecure: true - # username: "admin" - # password: "admin" - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-raw - # otel-service-map-pipeline: - # workers: 5 - # delay: 3000 - # source: - # pipeline: - # name: "otel-trace-pipeline" - # processor: - # - service_map: - # # The window duration is the maximum length of time the data prepper stores the most recent trace data to evaluvate service-map relationships. - # # The default is 3 minutes, this means we can detect relationships between services from spans reported in last 3 minutes. - # # Set higher value if your applications have higher latency. - # window_duration: 180 - # buffer: - # bounded_blocking: - # # buffer_size is the number of ExportTraceRequest from otel-collector the data prepper should hold in memeory. - # # We recommend to keep the same buffer_size for all pipelines. - # # Make sure you configure sufficient heap - # # default value is 12800 - # buffer_size: 25600 - # # This is the maximum number of request each worker thread will process within the delay. - # # Default is 200. - # # Make sure buffer_size >= workers * batch_size - # batch_size: 400 - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: trace-analytics-service-map - # #index: otel-v1-apm-span-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 - # otel-metrics-pipeline: - # workers: 8 - # delay: 3000 - # source: - # otel_metrics_source: - # health_check_service: true - # ssl: false - # buffer: - # bounded_blocking: - # buffer_size: 1024 # max number of records the buffer accepts - # batch_size: 1024 # max number of records the buffer drains after each read - # processor: - # - otel_metrics: - # calculate_histogram_buckets: true - # calculate_exponential_histogram_buckets: true - # exponential_histogram_max_allowed_scale: 10 - # flatten_attributes: false - # sink: - # - opensearch: - # hosts: ["https://opensearch-cluster-master:9200"] - # username: "admin" - # password: "admin" - # insecure: true - # index_type: custom - # index: metrics-%{yyyy.MM.dd} - # #max_retries: 20 - # bulk_size: 4 + ## Provide your pipeline configuration here if 'enabled' is set to true. See documentation for more advanced pipelines + #simple-sample-pipeline: + # workers: 2 # the number of workers + # delay: 5000 # in milliseconds, how long workers wait between read attempts + # source: + # random: {} + # buffer: + # bounded_blocking: + # buffer_size: 1024 # max number of records the buffer accepts + # batch_size: 256 # max number of records the buffer drains after each read + # processor: + # - string_converter: + # upper_case: true + # sink: + # - stdout: {} # -- Data Prepper ports ports: