diff --git a/.github/workflows/benchmarks.yaml b/.github/workflows/benchmarks.yaml new file mode 100644 index 00000000..9a927052 --- /dev/null +++ b/.github/workflows/benchmarks.yaml @@ -0,0 +1,92 @@ +on: [push] + +name: SQLServer NDC component benchmarks + +permissions: + contents: write + deployments: write + +jobs: + benchmark: + name: Benchmark + runs-on: ubuntu-latest + steps: + - name: Checkout 🛎️ + uses: actions/checkout@v4 + + - name: Install Nix ❄ + uses: cachix/install-nix-action@v23 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up the Nix Cache 🔌 + uses: cachix/cachix-action@v12 + with: + name: hasura-v3-dev + authToken: ${{ secrets.CACHIX_AUTH_TOKEN }} + + - name: Build the Docker image 🔨 + run: | + docker load < $(nix build --no-link --print-out-paths '.#docker') + + - name: Start dependencies ▶️ + run: | + cd benchmarks/component + docker compose up --detach --wait sqlserver grafana + + - name: Generate the deployment configuration 🚧 + run: | + set -e -u -o pipefail + cd benchmarks/component + mkdir -p generated + docker compose up --detach --wait agent-configuration + CONFIGURATION_SERVER_PORT="$(docker compose port agent-configuration 9100 | sed 's/.\+://')" + CONFIGURATION_SERVER="localhost:${CONFIGURATION_SERVER_PORT}" + SQLSERVER_CONNECTION_STRING="DRIVER={ODBC Driver 18 for SQL Server};SERVER=sqlserver,1433;Uid=SA;Database=Chinook;Pwd=Password!" + ../../scripts/new-configuration.sh "$CONFIGURATION_SERVER" "${SQLSERVER_CONNECTION_STRING}" \ + | tee ./generated/deployment.json + docker compose down agent-configuration + + - name: Run benchmarks 🏃 + run: | + cd benchmarks/component + for benchmark in $(ls benchmarks); do + echo "Starting agent..." + docker compose up --wait agent + echo "Running ${benchmark}..." + docker compose run --rm benchmark run "/benchmarks/$benchmark" + echo "Saving metrics for ${benchmark}..." + ./metrics.sh localhost:8100 ${benchmark} + echo "Stopping agent..." + docker compose down agent + done + + - name: Extract summaries ⛏️ + run: | + ./benchmarks/component/summarize.sh | tee benchmark-results.json + + # We run these benchmarks on every push so we can easily see the results. + # However, we only store the results on `main`. + - name: Store benchmark result ⬆️ + if: github.ref == 'refs/heads/main' + uses: benchmark-action/github-action-benchmark@v1 + with: + name: Component benchmarks + tool: customSmallerIsBetter + output-file-path: benchmark-results.json + github-token: ${{ secrets.GITHUB_TOKEN }} + auto-push: true + alert-threshold: 200% + comment-on-alert: true + + # scream into Slack if something goes wrong + - name: Report Status + if: always() && github.ref == 'refs/heads/main' + uses: ravsamhq/notify-slack-action@v2 + with: + status: ${{ job.status }} + notify_when: failure + notification_title: "😧 Error on <{repo_url}|{repo}>" + message_format: "🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>" + env: + SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }} diff --git a/benchmarks/component/README.md b/benchmarks/component/README.md new file mode 100644 index 00000000..19f8d254 --- /dev/null +++ b/benchmarks/component/README.md @@ -0,0 +1,46 @@ +# Component benchmarks + +This is the benchmark suite for the SQLServer data connector. + +Running `run.sh` with a benchmark name as an argument will: + +1. build the SQLServer data connector Docker image, +2. start the database with Chinook data, +3. start the agent using an associated deployment, and +4. run a benchmark using k6. + +Running without arguments will list available benchmarks. + +Everything is run through Docker Compose. + +The Docker image for the agent is built with Nix. If you haven't built with Nix +before (or it's been a while), this may take some time at first. + +## Requirements + +1. _Nix_, to build the Docker image + 1. Install [Nix](https://nixos.org/download.html) + 2. Configure Nix by adding the following line to `~/.config/nix/nix.conf`: + ``` + extra-experimental-features = flakes nix-command + ``` +2. _Docker_ and _Docker Compose_, to run the containers (see the root README) + +## Viewing the benchmark results + +When the benchmarks finish, the results will be printed. + +There is a Grafana dashboard which can be viewed as follows: + +1. Open [http://localhost:64300][]. +2. Open the menu on the left and choose "Dashboards". +3. Choose the "Test Result" dashboard. + +## Adding a benchmark + +You can add a benchmark by copying one of the files in the "benchmarks" +subdirectory and altering it. + +Please make sure that the name of the file corresponds to the `testid`. + +For further information, consult the [k6 documentation](https://k6.io/docs/). diff --git a/benchmarks/component/benchmarks/select-by-pk.js b/benchmarks/component/benchmarks/select-by-pk.js new file mode 100644 index 00000000..5f0027ec --- /dev/null +++ b/benchmarks/component/benchmarks/select-by-pk.js @@ -0,0 +1,67 @@ +import { check } from "k6"; +import http from "k6/http"; +import { newSummaryHandler } from "../common.js"; + +const testid = "select-by-pk"; +const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100"; +const url = `http://${agentSocket}/query`; +const data = { + collection: "Album", + query: { + fields: { + id: { type: "column", column: "AlbumId", arguments: {} }, + }, + where: { + type: "binary_comparison_operator", + column: { + type: "column", + name: "AlbumId", + path: [], + }, + operator: { + type: "equal", + }, + value: { + type: "scalar", + value: 1, + }, + }, + }, + arguments: {}, + collection_relationships: {}, +}; + +export default function () { + const response = http.post(url, JSON.stringify(data), { + headers: { + "Content-Type": "application/json", + }, + }); + + check(response, { + "status is 200": (r) => r.status == 200, + }); +} + +export const handleSummary = newSummaryHandler(testid); + +export const options = { + tags: { + testid, + }, + scenarios: { + short_sustained: { + executor: "constant-vus", + vus: 100, + duration: "10s", + }, + }, + thresholds: { + checks: [ + { + threshold: "rate == 1", + abortOnFail: true, + }, + ], + }, +}; diff --git a/benchmarks/component/benchmarks/select-variables.js b/benchmarks/component/benchmarks/select-variables.js new file mode 100644 index 00000000..e5833549 --- /dev/null +++ b/benchmarks/component/benchmarks/select-variables.js @@ -0,0 +1,89 @@ +import { check } from "k6"; +import http from "k6/http"; +import { newSummaryHandler } from "../common.js"; + +const testid = "select-variables"; +const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100"; +const url = `http://${agentSocket}/query`; +const data = { + collection: "Album", + query: { + fields: { + Title: { + type: "column", + column: "Title", + arguments: {}, + }, + }, + where: { + type: "binary_comparison_operator", + column: { + type: "column", + name: "Title", + path: [], + }, + operator: { + type: "other", + name: "_like", + }, + value: { + type: "variable", + name: "search", + }, + }, + }, + arguments: {}, + collection_relationships: {}, + variables: [ + { + search: "%Garage%", + }, + { + search: "%Good%", + }, + { + search: "%Rock%", + }, + { + search: "%Dog%", + }, + { + search: "%Log%", + }, + ], +}; + +export default function () { + const response = http.post(url, JSON.stringify(data), { + headers: { + "Content-Type": "application/json", + }, + }); + + check(response, { + "status is 200": (r) => r.status == 200, + }); +} + +export const handleSummary = newSummaryHandler(testid); + +export const options = { + tags: { + testid, + }, + scenarios: { + short_sustained: { + executor: "constant-vus", + vus: 100, + duration: "10s", + }, + }, + thresholds: { + checks: [ + { + threshold: "rate == 1", + abortOnFail: true, + }, + ], + }, +}; diff --git a/benchmarks/component/benchmarks/select-where.js b/benchmarks/component/benchmarks/select-where.js new file mode 100644 index 00000000..dd4bf0a1 --- /dev/null +++ b/benchmarks/component/benchmarks/select-where.js @@ -0,0 +1,70 @@ +import { check } from "k6"; +import http from "k6/http"; +import { newSummaryHandler } from "../common.js"; + +const testid = "select-where"; +const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100"; +const url = `http://${agentSocket}/query`; +const data = { + collection: "Album", + query: { + fields: { + id: { type: "column", column: "AlbumId", arguments: {} }, + title: { type: "column", column: "Title", arguments: {} }, + artist_id: { type: "column", column: "ArtistId", arguments: {} }, + }, + where: { + type: "binary_comparison_operator", + column: { + type: "column", + name: "Title", + path: [], + }, + operator: { + type: "other", + name: "_like", + }, + value: { + type: "scalar", + value: "%a%", + }, + }, + }, + arguments: {}, + collection_relationships: {}, +}; + +export default function () { + const response = http.post(url, JSON.stringify(data), { + headers: { + "Content-Type": "application/json", + }, + }); + + check(response, { + "status is 200": (r) => r.status == 200, + }); +} + +export const handleSummary = newSummaryHandler(testid); + +export const options = { + tags: { + testid, + }, + scenarios: { + short_sustained: { + executor: "constant-vus", + vus: 100, + duration: "10s", + }, + }, + thresholds: { + checks: [ + { + threshold: "rate == 1", + abortOnFail: true, + }, + ], + }, +}; diff --git a/benchmarks/component/benchmarks/select.js b/benchmarks/component/benchmarks/select.js new file mode 100644 index 00000000..d555a47f --- /dev/null +++ b/benchmarks/component/benchmarks/select.js @@ -0,0 +1,54 @@ +import { check } from "k6"; +import http from "k6/http"; +import { newSummaryHandler } from "../common.js"; + +const testid = "select"; +const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100"; +const url = `http://${agentSocket}/query`; +const data = { + collection: "Album", + query: { + fields: { + id: { type: "column", column: "AlbumId", arguments: {} }, + title: { type: "column", column: "Title", arguments: {} }, + artist_id: { type: "column", column: "ArtistId", arguments: {} }, + }, + }, + arguments: {}, + collection_relationships: {}, +}; + +export default function () { + const response = http.post(url, JSON.stringify(data), { + headers: { + "Content-Type": "application/json", + }, + }); + + check(response, { + "status is 200": (r) => r.status == 200, + }); +} + +export const handleSummary = newSummaryHandler(testid); + +export const options = { + tags: { + testid, + }, + scenarios: { + short_sustained: { + executor: "constant-vus", + vus: 100, + duration: "10s", + }, + }, + thresholds: { + checks: [ + { + threshold: "rate == 1", + abortOnFail: true, + }, + ], + }, +}; diff --git a/benchmarks/component/common.js b/benchmarks/component/common.js new file mode 100644 index 00000000..57f17b70 --- /dev/null +++ b/benchmarks/component/common.js @@ -0,0 +1,21 @@ +// Sadly, k6 does not export its default text summarizer directly, +// instead making it available as a separate library. +// We need to import it from there, via their CDN. +import { textSummary } from "https://jslib.k6.io/k6-summary/0.0.3/index.js"; + +// This constructs a summary handler, injecting the test ID. +export function newSummaryHandler(testid) { + return function handleSummary(data) { + const summaries = { + // We always want to print a text summary. + stdout: textSummary(data, { indent: " ", enableColors: true }), + }; + const outputDirectory = __ENV.OUTPUT_DIRECTORY; + if (outputDirectory) { + const summaryFile = `${outputDirectory}/summaries/${testid}__${new Date().toISOString()}.json`; + // If there is an output directory provided, we also serialize the summary to JSON and write it there. + summaries[summaryFile] = JSON.stringify(data); + } + return summaries; + }; +} diff --git a/benchmarks/component/docker-compose.yaml b/benchmarks/component/docker-compose.yaml new file mode 100644 index 00000000..0b8843a8 --- /dev/null +++ b/benchmarks/component/docker-compose.yaml @@ -0,0 +1,142 @@ +version: "3.6" + +name: ndc-sqlserver-component-benchmarks + +services: + benchmark: + image: grafana/k6 + environment: + AGENT_SOCKET: "agent:8100" + OUTPUT_DIRECTORY: "/home/k6/output" + K6_OUT: "experimental-prometheus-rw" + K6_PROMETHEUS_RW_SERVER_URL: "http://prometheus:9090/api/v1/write" + K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM: "true" + volumes: + - type: volume + source: benchmark_output + target: /home/k6/output + - type: bind + source: ./benchmarks + target: /benchmarks + read_only: true + - type: bind + source: ./common.js + target: /common.js + read_only: true + depends_on: + agent: + condition: service_started + prometheus: + condition: service_started + benchmark-output-init: + condition: service_completed_successfully + + benchmark-output-init: + image: grafana/k6 + entrypoint: + - sh + - -c + command: + - | + /bin/mkdir -p /home/k6/output/summaries + /bin/chown -R k6: /home/k6/output + user: root + volumes: + - type: volume + source: benchmark_output + target: /home/k6/output + + sqlserver: + extends: + file: ../../docker-compose.yaml + service: sqlserver + ports: + - 1433 + + agent: + image: ghcr.io/hasura/ndc-sqlserver:dev + command: + - serve + init: true + ports: + - "8100:8100" + environment: + CONFIGURATION_FILE: "/deployment.json" + # we don't care about traces right now, and the benchmarks flood the batch buffer + OTEL_TRACES_SAMPLER: "always_off" + PORT: 8100 + volumes: + - type: bind + source: ./generated/deployment.json + target: /deployment.json + read_only: true + healthcheck: + test: + - CMD + - ndc-sqlserver + - check-health + interval: 1s + timeout: 1s + retries: 30 + depends_on: + sqlserver: + condition: service_started + + agent-configuration: + image: ghcr.io/hasura/ndc-sqlserver:dev + command: + - configuration + - serve + init: true + ports: + - 9100 + environment: + PORT: 9100 + healthcheck: + test: + - CMD + - ndc-sqlserver + - check-health + - --port=9100 + interval: 1s + timeout: 1s + retries: 30 + depends_on: + sqlserver: + condition: service_started + + # k6 will push statistics to Prometheus + prometheus: + image: prom/prometheus + command: + - --config.file=/etc/prometheus/prometheus.yml + - --web.enable-remote-write-receiver + - --enable-feature=native-histograms + ports: + - 9090 + volumes: + - type: volume + source: prometheus + target: /prometheus + + # Grafana will visualize k6 statistics + grafana: + image: grafana/grafana + ports: + - 64300:3000 + environment: + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_BASIC_ENABLED=false + volumes: + - type: bind + source: ../grafana + target: /etc/grafana/provisioning + depends_on: + prometheus: + condition: service_started + +volumes: + benchmark_output: + metrics_output: + prometheus: diff --git a/benchmarks/component/metrics.sh b/benchmarks/component/metrics.sh new file mode 100755 index 00000000..017155b6 --- /dev/null +++ b/benchmarks/component/metrics.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -e +set -u +set -o pipefail + +# given an address like `localhost:8100`, call metrics endpoint and extract +# useful things like the time our agent spends doing work that isn't query execution or +# waiting for connections + +server_address="$1" +job_name="$2" + +# make a temp folder to put our rubbish in +directory="$(mktemp)" +rm -f "$directory" +mkdir "$directory" +trap 'rm -rf "$directory"' EXIT + +entire_address="http://${server_address}/metrics" + +# get metrics +curl $entire_address > $directory/metrics.txt + +REQUEST_TIME=$(cat $directory/metrics.txt \ + | sed -n -e '/^ndc_sqlserver_query_total_time_sum/p' \ + | cut -d' ' -f2 ) + +CONNECTION_ACQUISITION_TIME=$(cat $directory/metrics.txt \ + | sed -n -e '/^ndc_sqlserver_connection_acquisition_wait_time_sum/p' \ + | cut -d' ' -f2 ) + +QUERY_TIME=$(cat $directory/metrics.txt \ + | sed -n -e '/^ndc_sqlserver_query_execution_time_sum/p' \ + | cut -d' ' -f2 ) + +REQUESTS_COUNT=$(cat $directory/metrics.txt \ + | sed -n -e '/^ndc_sqlserver_query_total_time_count/p' \ + | cut -d' ' -f2 ) + +REQUEST_TIME_MS=$(echo "scale=5;$REQUEST_TIME * 1000" | bc) +CONNECT_TIME_MS=$(echo "scale=5;$CONNECTION_ACQUISITION_TIME * 1000" | bc) +QUERY_TIME_MS=$(echo "scale=5;$QUERY_TIME * 1000" | bc) + +PROCESSING_TIME_MS=$(echo "scale=5;$REQUEST_TIME_MS - $CONNECT_TIME_MS - $QUERY_TIME_MS" | bc) + +OUTPUT_JSON=$(jq -r --null-input \ + --arg request "$REQUEST_TIME_MS" \ + --arg connection "$CONNECT_TIME_MS" \ + --arg query "$QUERY_TIME_MS" \ + --arg processing "$PROCESSING_TIME_MS" \ + '{ "request": $request, "connection_acquisition": $connection, "query_execution": $query, "processing": $processing }') + +output_file_name="${directory}/metrics-${job_name}.json" + +# save JSON in folder +echo $OUTPUT_JSON > $output_file_name + +# copy $directory into Docker volume 'metrics_output' +tar c -C "$directory" . \ + | docker run \ + --rm \ + --interactive \ + --mount 'type=volume,source=sqlserver-ndc-component-benchmarks_metrics_output,target=/input' \ + busybox \ + tar x -C /input . + diff --git a/benchmarks/component/run.sh b/benchmarks/component/run.sh new file mode 100755 index 00000000..e7baf267 --- /dev/null +++ b/benchmarks/component/run.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +# prints its arguments to STDERR in green +function info { + echo >&2 "> $(tput setaf 2)$*$(tput sgr0)" +} + +function stop { + if [[ -e ./agent.pid ]]; then + info 'Stopping the agent' + AGENT_PID="$(< ./agent.pid)" + kill "$AGENT_PID" || : + rm ./agent.pid + fi +} + +if [[ $# -eq 0 ]]; then + echo >&2 "Usage: ${BASH_SOURCE[0]} BENCHMARK [k6 args...]" + echo >&2 + echo >&2 "Benchmarks:" + ls ./benchmarks | sed 's/^/ - /' + exit 2 +fi + +BENCHMARK="$1" +shift +if [[ ! -f "./benchmarks/$BENCHMARK" ]]; then + echo >&2 "ERROR: Unknown benchmark: $BENCHMARK" + echo >&2 + echo >&2 "Benchmarks:" + ls ./benchmarks | sed 's/^/ - /' + exit 1 +fi + +trap stop EXIT INT QUIT TERM +./start.sh + +info 'Running the benchmarks' +export K6_OUT="experimental-prometheus-rw" +export K6_PROMETHEUS_RW_SERVER_URL="http://$(docker compose port prometheus 9090)/api/v1/write" +export K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM="true" +k6 run "./benchmarks/$BENCHMARK" "$@" + +info 'Printing metrics' +curl -fsS http://localhost:8100/metrics diff --git a/benchmarks/component/start.sh b/benchmarks/component/start.sh new file mode 100755 index 00000000..d1fd1248 --- /dev/null +++ b/benchmarks/component/start.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +set -e +set -u +set -o pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +# prints its arguments to STDERR in green +function info { + echo >&2 "> $(tput setaf 2)$*$(tput sgr0)" +} + +if [[ -e ./agent.pid ]]; then + info 'Stopping the agent from a previous run' + AGENT_PID="$(< ./agent.pid)" + kill "$AGENT_PID" && wait "$AGENT_PID" || : + rm ./agent.pid +fi + +info 'Building the agent' +cargo build --release -p ndc-sqlserver + +info 'Starting the dependencies' +(cd ../.. && docker compose up --wait jaeger) # avoid port clobbering by re-using Jaeger +docker compose up --wait sqlserver grafana +SQLSERVER_SOCKET="$(docker compose port sqlserver 1433 | cut -f2 -d":")" + +info 'Generating the deployment configuration' +mkdir -p generated +cargo run -p ndc-sqlserver --quiet --release -- configuration serve & +AGENT_PID=$! +echo "$AGENT_PID" > ./agent.pid +../../scripts/wait-until --timeout=30 --report -- cargo run -p ndc-sqlserver --quiet -- check-health --port=9100 +if ! kill -0 "$AGENT_PID"; then + echo >&2 'The agent stopped abruptly. Take a look at agent.log for details.' + exit 1 +fi + +SQLSERVER_CONNECTION_STRING="DRIVER={ODBC Driver 18 for SQL Server};SERVER=127.0.0.1,${SQLSERVER_SOCKET};Uid=SA;Database=Chinook;Pwd=Password!" + +../../scripts/new-configuration.sh localhost:9100 "${SQLSERVER_CONNECTION_STRING}" \ + > ./generated/deployment.json +kill "$AGENT_PID" && wait "$AGENT_PID" || : +rm -f ./agent.pid + +info 'Starting the agent' +if nc -z localhost 8100; then + echo >&2 'ERROR: There is already an agent running on port 8100.' + exit 1 +fi + +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT='http://localhost:4317' \ + OTEL_SERVICE_NAME='ndc-sqlserver' \ + cargo run -p ndc-sqlserver --quiet --release -- \ + serve --configuration=./generated/deployment.json \ + >& agent.log & +AGENT_PID=$! +echo "$AGENT_PID" > ./agent.pid +echo >&2 "The agent is running with PID ${AGENT_PID}" +../../scripts/wait-until --timeout=30 --report -- cargo run -p ndc-sqlserver --quiet -- check-health + +if ! kill -0 "$AGENT_PID"; then + echo >&2 'The agent stopped abruptly. Take a look at agent.log for details.' + exit 1 +fi diff --git a/benchmarks/component/summarize.sh b/benchmarks/component/summarize.sh new file mode 100755 index 00000000..c65c30f3 --- /dev/null +++ b/benchmarks/component/summarize.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Summarizes the stored benchmark results. +# Intended for use with the "continuous benchmark" GitHub action, but might also +# be useful elsewhere. + +set -e +set -u +set -o pipefail + +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" + +# prints its arguments to STDERR in green +function info { + echo >&2 "> $(tput setaf 2)$*$(tput sgr0)" +} + +directory="$(mktemp)" +rm -f "$directory" +mkdir "$directory" +trap 'rm -rf "$directory"' EXIT + +# the output of the K6 jobs is sitting in the `benchmark_output` volume, +# so we mount it in an arbitrary container which we use to copy it out. +docker run \ + --rm \ + --mount 'type=volume,source=ndc-sqlserver-component-benchmarks_benchmark_output,target=/output' \ + busybox \ + tar c -C /output . \ + | tar x -C "$directory" + +# the output of the metrics for each job is sitting in the `metrics_output` volume, +# so we mount it in an arbitrary container which we use to copy it out. +docker run \ + --rm \ + --mount 'type=volume,source=sqlserver-ndc-component-benchmarks_metrics_output,target=/output' \ + busybox \ + tar c -C /output . \ + | tar x -C "$directory" + +for summary_file in "$directory/summaries/"*; do + name="$(basename "$summary_file" | sed 's/__.*//')" + metrics_file="${directory}/metrics-${name}.js.json" + query_time=$(jq -c '.query_execution' "$metrics_file") + connection_time=$(jq -c '.connection_acquisition' "$metrics_file") + processing_time=$(jq -c '.processing' "$metrics_file") + jq -c --arg name "$name" \ + --arg query_time "$query_time" \ + --arg connection_time "$connection_time" \ + --arg processing_time "$processing_time" \ + '.metrics | [ + {"name": ($name + " - median"), "unit": "ms", "value": .http_req_duration.values.med}, + {"name": ($name + " - p(95)"), "unit": "ms", "value": .http_req_duration.values."p(95)"}, + {"name": ($name + " - connection acquisition time"), "unit": "ms", "value": (($connection_time | fromjson | tonumber) / .http_reqs.values.count ) }, + {"name": ($name + " - request time - (query + acquisition)"), "unit": "ms", "value": (.http_req_duration.values.avg - (($query_time | fromjson | tonumber) / .http_reqs.values.count) - (($connection_time | fromjson | tonumber) / .http_reqs.values.count)) }, + {"name": ($name + " - processing time"), "unit": "ms", "value": (($processing_time | fromjson | tonumber) / .http_reqs.values.count ) } + ]' \ + "$summary_file" +done | jq -s 'flatten(1)' diff --git a/benchmarks/grafana/README.md b/benchmarks/grafana/README.md new file mode 100644 index 00000000..31ff34d1 --- /dev/null +++ b/benchmarks/grafana/README.md @@ -0,0 +1,3 @@ +These Grafana dashboards are optimized for k6. + +They were copied from https://github.com/grafana/xk6-output-prometheus-remote. diff --git a/benchmarks/grafana/dashboards/dashboard-apdex.json b/benchmarks/grafana/dashboards/dashboard-apdex.json new file mode 100644 index 00000000..8aaace29 --- /dev/null +++ b/benchmarks/grafana/dashboards/dashboard-apdex.json @@ -0,0 +1,768 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "k6 Test Result", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 118, + "panels": [], + "title": "Apdex Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 124, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "# Apdex\n\n[Apdex](https://en.wikipedia.org/wiki/Apdex) (_Application Performance Index_) allows us to monitor end-user satisfaction. \nIt is an open industry standard that estimates the end user’s satisfaction level on an application’s response time through a score between 0 and 1.\n\nThe Apdex score is calculated based on your required SLA (_Service-Level Agreement_) where you can define a response time threshold of T seconds, where all responses handled in T or less seconds satisfy the end user.", + "mode": "markdown" + }, + "pluginVersion": "9.4.3", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "from": 0.94, + "result": { + "color": "green", + "index": 0 + }, + "to": 1 + }, + "type": "range" + }, + { + "options": { + "from": 0.85, + "result": { + "color": "super-light-green", + "index": 1 + }, + "to": 0.93 + }, + "type": "range" + }, + { + "options": { + "from": 0.7, + "result": { + "color": "super-light-orange", + "index": 2 + }, + "to": 0.85 + }, + "type": "range" + }, + { + "options": { + "from": 0.5, + "result": { + "color": "yellow", + "index": 3 + }, + "to": 0.7 + }, + "type": "range" + }, + { + "options": { + "from": 0, + "result": { + "color": "red", + "index": 4 + }, + "to": 0.5 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 122, + "interval": "1", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(histogram_fraction($T, $T*4, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}) + (histogram_fraction(0, $T, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"})/2))/\n\n\nhistogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"})", + "hide": false, + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "from": 0.94, + "result": { + "color": "green", + "index": 0, + "text": "Excellent" + }, + "to": 1 + }, + "type": "range" + }, + { + "options": { + "from": 0.85, + "result": { + "color": "super-light-green", + "index": 1, + "text": "Good" + }, + "to": 0.93 + }, + "type": "range" + }, + { + "options": { + "from": 0.7, + "result": { + "color": "super-light-orange", + "index": 2, + "text": "Fair" + }, + "to": 0.85 + }, + "type": "range" + }, + { + "options": { + "from": 0.5, + "result": { + "color": "yellow", + "index": 3, + "text": "Poor" + }, + "to": 0.7 + }, + "type": "range" + }, + { + "options": { + "from": 0, + "result": { + "color": "red", + "index": 4, + "text": "Unacceptable" + }, + "to": 0.5 + }, + "type": "range" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 126, + "interval": "1", + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "value_and_name" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "(histogram_fraction($T, $T*4, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}) + (histogram_fraction(0, $T, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"})/2))/\n\n\nhistogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"})", + "hide": false, + "legendFormat": "{{name}}", + "range": true, + "refId": "A" + } + ], + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "**Satisfied:** Response time less than or equal to T seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 13 + }, + "id": 120, + "interval": "1", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_fraction(0, $T, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"})>0", + "hide": false, + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "title": "Satisfied requests", + "transformations": [ + { + "id": "calculateField", + "options": {} + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "- **Tolerating:** Response time between T seconds and 4T seconds.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 13 + }, + "id": 125, + "interval": "1", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_fraction($T, $T*4, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"})>0", + "hide": false, + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "title": "Tolerating requests", + "transformations": [ + { + "id": "calculateField", + "options": {} + } + ], + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "**Frustrating:** Response time greater than 4 T seconds + Failed.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 13 + }, + "id": 121, + "interval": "1", + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_fraction($T*4, 9999999, ($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"}))*histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"true\"})>0", + "hide": false, + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "histogram_count($metrics{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\",expected_response=\"false\"})>0", + "hide": false, + "instant": true, + "legendFormat": "{{name}}", + "range": false, + "refId": "B" + } + ], + "title": "Frustrating requests", + "transformations": [ + { + "id": "calculateField", + "options": {} + } + ], + "type": "bargauge" + } + ], + "refresh": "5s", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": ["prometheus", "k6"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(testid)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "testid", + "options": [], + "query": { + "query": "label_values(testid)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(scenario)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "scenario", + "options": [], + "query": { + "query": "label_values(scenario)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "http.*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(url)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "url", + "options": [], + "query": { + "query": "label_values(url)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "selected": false, + "text": "0.1", + "value": "0.1" + }, + "hide": 0, + "label": " [T]arget time (seconds) Apdex", + "name": "T", + "options": [ + { + "selected": false, + "text": "0.3", + "value": "0.3" + } + ], + "query": "0.1", + "skipUrlSync": false, + "type": "textbox" + }, + { + "current": { + "selected": false, + "text": "k6_http_req_duration_seconds", + "value": "k6_http_req_duration_seconds" + }, + "description": "Metrics for apdex", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "metrics", + "options": [ + { + "selected": false, + "text": "k6_http_req_waiting_seconds", + "value": "k6_http_req_waiting_seconds" + }, + { + "selected": true, + "text": "k6_http_req_duration_seconds", + "value": "k6_http_req_duration_seconds" + } + ], + "query": "k6_http_req_waiting_seconds, k6_http_req_duration_seconds", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Apdex Overview", + "uid": "01npcT44d", + "version": 2, + "weekStart": "" +} diff --git a/benchmarks/grafana/dashboards/dashboard-custom-metrics.json b/benchmarks/grafana/dashboards/dashboard-custom-metrics.json new file mode 100644 index 00000000..ea9791ec --- /dev/null +++ b/benchmarks/grafana/dashboards/dashboard-custom-metrics.json @@ -0,0 +1,503 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus-in-docker", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "gauge", + "name": "Gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.3.2" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "k6 Test Result", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 109, + "panels": [], + "title": "Example Custom Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 111, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "k6_my_counter_total{testid=~\"$testid\"}", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Counter", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 112, + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "k6_my_gauge{testid=~\"$testid\"}", + "format": "time_series", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "gouge", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "red", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 113, + "options": { + "orientation": "auto", + "reduceOptions": { + "calcs": ["mean"], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg(k6_my_rate_rate{testid=~\"$testid\"})", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Rate", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 114, + "interval": "1s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "k6_my_trend{testid=~\"$testid\"}", + "hide": false, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "trends wip", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": ["prometheus", "k6"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus-in-docker", + "value": "prometheus-in-docker" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": "", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(testid)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "testid", + "options": [], + "query": { + "query": "label_values(testid)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(scenario)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "scenario", + "options": [], + "query": { + "query": "label_values(scenario)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "http.*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(url)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "url", + "options": [], + "query": { + "query": "label_values(url)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Example Custom Metrics", + "uid": "01npcT44a", + "version": 1, + "weekStart": "" +} diff --git a/benchmarks/grafana/dashboards/dashboard-results.json b/benchmarks/grafana/dashboards/dashboard-results.json new file mode 100644 index 00000000..29ceba2a --- /dev/null +++ b/benchmarks/grafana/dashboards/dashboard-results.json @@ -0,0 +1,4049 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "stat", + "name": "Stat", + "version": "" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "k6 Test Result", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 47, + "panels": [], + "title": "Performance Overview", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "null", + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 119, + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 4, + "showHeader": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "k6_http_reqs_total{testid=~\"$testid\", name=~\"$url\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Tests in this Time Range", + "transformations": [ + { + "id": "joinByLabels", + "options": { + "value": "expected_response" + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true, + "method": true, + "proto": true, + "scenario": true, + "status": true, + "tls_version": true, + "true": true, + "url": true + }, + "indexByName": { + "method": 2, + "name": 1, + "proto": 3, + "scenario": 4, + "status": 5, + "testid": 0, + "tls_version": 6, + "true": 8, + "url": 7 + }, + "renameByName": {} + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "name": { + "aggregations": [], + "operation": "groupby" + }, + "testid": { + "aggregations": [], + "operation": "groupby" + } + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 49, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_http_reqs_total{testid=~\"$testid\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Requests Made", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + }, + { + "color": "red", + "value": 0 + } + ] + }, + "unit": "reqs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 4, + "y": 6 + }, + "id": 71, + "interval": "1s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "value_and_name" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "HTTP Failures", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 51, + "interval": "1s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (deriv(k6_http_reqs_total{testid=~\"$testid\"}[$__range]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Peak RPS", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 6 + }, + "id": 52, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["max"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "histogram_quantile(0.95, sum by(testid) (rate($metrics{testid=~\"$testid\"}[$__rate_interval]))) ", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "P95 Response Time", + "transformations": [], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 6 + }, + "id": 63, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_data_received_total{testid=~\"$testid\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Data Received", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 20, + "y": 6 + }, + "id": 64, + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_data_sent_total{testid=~\"$testid\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Data Sent", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Response Time (avg) - " + }, + "properties": [ + { + "id": "displayName", + "value": "Response Time (avg) - without testid" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "color", + "value": { + "mode": "fixed" + } + }, + { + "id": "custom.axisLabel", + "value": "VUs" + }, + { + "id": "custom.scaleDistribution", + "value": { + "log": 10, + "type": "log" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.axisLabel", + "value": "RPS" + }, + { + "id": "custom.lineWidth", + "value": 3 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 5 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.axisPlacement", + "value": "hidden" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.fillOpacity", + "value": 18 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.axisLabel", + "value": "Response Time" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active VUs - " + }, + "properties": [ + { + "id": "displayName", + "value": "Active VUs - without testid" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests Rate - " + }, + "properties": [ + { + "id": "displayName", + "value": "Requests Rate - without testid" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed Requests Rate - " + }, + "properties": [ + { + "id": "displayName", + "value": "Failed Requests Rate - without testid" + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 14, + "interval": "5", + "options": { + "legend": { + "calcs": ["min", "mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "timezones": ["browser"], + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "k6_vus{testid=~\"$testid\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Active VUs - {{testid}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(testid) (rate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Requests Rate - {{testid}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by(testid) (sum by(testid) (rate(k6_http_reqs_total{testid=~\"$testid\", expected_response=\"false\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "Failed Requests Rate - {{testid}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(histogram_sum(rate($metrics{testid=~\"$testid\"}[$__rate_interval]))\n/\nhistogram_count(rate($metrics{testid=~\"$testid\"}[$__rate_interval]))) ", + "hide": false, + "legendFormat": "Response Time (avg) - {{testid}}", + "range": true, + "refId": "C" + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "text", + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": " -" + }, + "properties": [ + { + "id": "displayName", + "value": "without testid -" + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 24 + }, + "id": 85, + "interval": "1s", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_iterations_total{testid=~\"$testid\"})", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Iteration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Received - " + }, + "properties": [ + { + "id": "displayName", + "value": "Received - without testid" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Sent - " + }, + "properties": [ + { + "id": "displayName", + "value": "Sent - without testid" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 9, + "x": 4, + "y": 24 + }, + "id": 18, + "interval": "5s", + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(sum by(testid)(irate(k6_data_received_total{testid=~\"$testid\"}[$__rate_interval])))", + "interval": "", + "legendFormat": "Received - {{testid}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(sum by(testid)(irate(k6_data_sent_total{testid=~\"$testid\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Sent - {{testid}}", + "range": true, + "refId": "B" + } + ], + "title": "Data flow", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/[2]0./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/[3]0./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-purple", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/[4][0-9]./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-orange", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/[5][0-6]./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/[1]0./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "super-light-purple", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 13, + "y": 24 + }, + "id": 78, + "interval": "5", + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "100 * (sum by(status, testid) (k6_http_reqs_total{testid=~\"$testid\"}) / on() group_left() sum(k6_http_reqs_total{testid=~\"$testid\"}) )", + "hide": false, + "instant": false, + "legendFormat": "{{status}} - {{testid}}", + "range": true, + "refId": "A" + } + ], + "title": "Code Status Rate", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "/([2]0. -) $/", + "renamePattern": "$1 without testid " + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "/([3]0. -) $/", + "renamePattern": "$1 without testid " + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "/([4][0-9]. -) $/", + "renamePattern": "$1 without testid " + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "/([5][0-6]. -) $/", + "renamePattern": "$1 without testid " + } + }, + { + "id": "renameByRegex", + "options": { + "regex": "/([1]0. -) $/", + "renamePattern": "$1 without testid " + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "palette-classic" + }, + "mappings": [ + { + "options": { + "match": "null+nan", + "result": { + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 29 + }, + "id": 92, + "interval": "1s", + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "sum by(testid) (k6_dropped_iterations_total{testid=~\"$testid\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{testid}} -", + "range": true, + "refId": "A" + } + ], + "title": "Dropped Iteration ", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0.5 + }, + { + "color": "#6ED0E0", + "value": 0.9 + }, + { + "color": "blue", + "value": 1 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Success Rate" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "empty", + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 13, + "x": 0, + "y": 34 + }, + "id": 104, + "interval": "1", + "options": { + "footer": { + "countRows": false, + "enablePagination": true, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 2, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "k6_checks_rate{testid=~\"$testid\"}", + "format": "table", + "instant": false, + "range": true, + "refId": "A" + } + ], + "title": "Checks", + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #A": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [] + }, + "__name__": { + "aggregations": [] + }, + "check": { + "aggregations": [], + "operation": "groupby" + }, + "expected_response": { + "aggregations": [], + "operation": "groupby" + }, + "scenario": { + "aggregations": [], + "operation": "groupby" + }, + "test_type": { + "aggregations": [] + }, + "testid": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Value (last)": false, + "testid": false + }, + "indexByName": { + "Value (lastNotNull)": 3, + "check": 1, + "scenario": 2, + "testid": 0 + }, + "renameByName": { + "Value #A (lastNotNull)": "P95 Response Time", + "Value #B (lastNotNull)": "Failed Request Count", + "Value #C (lastNotNull)": "AVG RPS", + "Value #D (lastNotNull)": "Iterations", + "Value #E (lastNotNull)": "Request Count", + "Value (last)": "Success Rate", + "Value (lastNotNull)": "Success Rate", + "scenario": "Scenario" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Request Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95 Response Time" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "AVG RPS" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 11, + "x": 13, + "y": 34 + }, + "id": 107, + "interval": "1", + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "test_type" + } + ] + }, + "pluginVersion": "9.4.3", + "title": "Thresholds WIP", + "transformations": [ + { + "disabled": true, + "id": "merge", + "options": {} + }, + { + "disabled": true, + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #A": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [] + }, + "expected_response": { + "aggregations": [], + "operation": "groupby" + }, + "scenario": { + "aggregations": [], + "operation": "groupby" + }, + "test_type": { + "aggregations": [] + }, + "testid": { + "aggregations": [] + } + } + } + }, + { + "disabled": true, + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A (lastNotNull)": 4, + "Value #B (lastNotNull)": 3, + "Value #C (lastNotNull)": 5, + "Value #D (lastNotNull)": 1, + "Value #E (lastNotNull)": 2, + "scenario": 0 + }, + "renameByName": { + "Value #A (lastNotNull)": "P95 Response Time", + "Value #B (lastNotNull)": "Failed Request Count", + "Value #C (lastNotNull)": "AVG RPS", + "Value #D (lastNotNull)": "Iterations", + "Value #E (lastNotNull)": "Request Count", + "Value (lastNotNull)": "Failed Request", + "scenario": "Scenario" + } + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "type": "prometheus" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 32, + "panels": [], + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "HTTP ", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "align": "center", + "cellOptions": { + "type": "color-text" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "status" + }, + "properties": [ + { + "id": "custom.cellOptions", + "value": { + "mode": "basic", + "type": "color-background" + } + }, + { + "id": "unit", + "value": "short" + }, + { + "id": "custom.width" + }, + { + "id": "mappings", + "value": [ + { + "options": { + "100": { + "color": "super-light-purple", + "index": 0, + "text": "100 Continue" + }, + "101": { + "color": "super-light-purple", + "index": 1, + "text": "101 Switching Protocols" + }, + "102": { + "color": "super-light-purple", + "index": 2, + "text": "102 Processing" + }, + "103": { + "color": "super-light-purple", + "index": 3, + "text": "103 Early Hints" + }, + "200": { + "color": "blue", + "index": 4, + "text": "200 OK" + }, + "201": { + "color": "blue", + "index": 5, + "text": "201 Created" + }, + "202": { + "color": "blue", + "index": 6, + "text": "202 Accepted" + }, + "203": { + "color": "blue", + "index": 7, + "text": "203 Non-Authoritative Information" + }, + "204": { + "color": "blue", + "index": 8, + "text": "204 No Content" + }, + "205": { + "color": "blue", + "index": 9, + "text": "205 Reset Content" + }, + "206": { + "color": "blue", + "index": 10, + "text": "206 Partial Content" + }, + "207": { + "color": "blue", + "index": 11, + "text": "207 Multi-Status" + }, + "208": { + "color": "blue", + "index": 12, + "text": "208 Already Reported " + }, + "226": { + "color": "blue", + "index": 13, + "text": "226 IM Used " + }, + "300": { + "color": "purple", + "index": 14, + "text": "300 Multiple Choices" + }, + "301": { + "color": "purple", + "index": 15, + "text": "301 Moved Permanently" + }, + "302": { + "color": "purple", + "index": 16, + "text": "302 Found" + }, + "303": { + "color": "purple", + "index": 17, + "text": "303 See Other" + }, + "304": { + "color": "purple", + "index": 18, + "text": "304 Not Modified" + }, + "305": { + "color": "purple", + "index": 19, + "text": "305 Use Proxy " + }, + "307": { + "color": "purple", + "index": 20, + "text": "307 Temporary Redirect" + }, + "308": { + "color": "purple", + "index": 21, + "text": "308 Permanent Redirect" + }, + "400": { + "color": "orange", + "index": 22, + "text": "400 Bad Request" + }, + "401": { + "color": "orange", + "index": 23, + "text": "401 Unauthorized" + }, + "402": { + "color": "orange", + "index": 24, + "text": "402 Payment Required" + }, + "403": { + "color": "orange", + "index": 25, + "text": "403 Forbidden" + }, + "404": { + "color": "orange", + "index": 26, + "text": "404 Not Found" + }, + "405": { + "color": "orange", + "index": 27, + "text": "405 Method Not Allowed" + }, + "406": { + "color": "orange", + "index": 28, + "text": "406 Not Acceptable" + }, + "407": { + "color": "orange", + "index": 29, + "text": "407 Proxy Authentication Required" + }, + "408": { + "color": "orange", + "index": 30, + "text": "408 Request Timeout" + }, + "409": { + "color": "orange", + "index": 31, + "text": "409 Conflict" + }, + "410": { + "color": "orange", + "index": 32, + "text": "410 Gone" + }, + "411": { + "color": "orange", + "index": 33, + "text": "411 Length Required" + }, + "412": { + "color": "orange", + "index": 34, + "text": "412 Precondition Failed" + }, + "413": { + "color": "orange", + "index": 35, + "text": "413 Payload Too Large" + }, + "414": { + "color": "orange", + "index": 36, + "text": "414 URI Too Long" + }, + "415": { + "color": "orange", + "index": 37, + "text": "415 Unsupported Media Type" + }, + "416": { + "color": "orange", + "index": 38, + "text": "416 Range Not Satisfiable" + }, + "417": { + "color": "orange", + "index": 39, + "text": "417 Expectation Failed" + }, + "418": { + "color": "orange", + "index": 40, + "text": "418 I'm a teapot" + }, + "421": { + "color": "orange", + "index": 41, + "text": "421 Misdirected Request" + }, + "422": { + "color": "orange", + "index": 42, + "text": "422 Unprocessable Entity" + }, + "423": { + "color": "orange", + "index": 43, + "text": "423 Locked " + }, + "424": { + "color": "orange", + "index": 44, + "text": "424 Failed Dependency" + }, + "425": { + "color": "orange", + "index": 45, + "text": "425 Too Early" + }, + "426": { + "color": "orange", + "index": 46, + "text": "426 Upgrade Required" + }, + "428": { + "color": "orange", + "index": 47, + "text": "428 Precondition Required" + }, + "429": { + "color": "orange", + "index": 48, + "text": "429 Too Many Requests" + }, + "431": { + "color": "orange", + "index": 49, + "text": "431 Request Header Fields Too Large" + }, + "451": { + "color": "orange", + "index": 50, + "text": "451 Unavailable For Legal Reasons431 Request Header Fields Too Large" + }, + "500": { + "color": "red", + "index": 51, + "text": "500 Internal Server Error" + }, + "501": { + "color": "red", + "index": 52, + "text": "501 Not Implemented" + }, + "502": { + "color": "red", + "index": 53, + "text": "502 Bad Gateway" + }, + "503": { + "color": "red", + "index": 54, + "text": "503 Service Unavailable" + }, + "504": { + "color": "red", + "index": 55, + "text": "504 Gateway Timeout" + }, + "505": { + "color": "red", + "index": 56, + "text": "505 HTTP Version Not Supported" + }, + "506": { + "color": "red", + "index": 57, + "text": "506 Variant Also Negotiates" + }, + "507": { + "color": "red", + "index": 58, + "text": "507 Insufficient Storage" + }, + "508": { + "color": "red", + "index": 59, + "text": "508 Loop Detected " + }, + "510": { + "color": "red", + "index": 60, + "text": "510 Not Extended" + }, + "511": { + "color": "red", + "index": 61, + "text": "511 Network Authentication Required" + } + }, + "type": "value" + } + ] + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "count" + }, + "properties": [ + { + "id": "unit", + "value": "short" + }, + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + }, + { + "id": "custom.width", + "value": 81 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min " + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 101 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mean" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 97 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "p95" + }, + "properties": [ + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.width", + "value": 92 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "custom.width", + "value": 132 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "name" + }, + "properties": [ + { + "id": "custom.width" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "scenario" + }, + "properties": [ + { + "id": "custom.width", + "value": 283 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "method" + }, + "properties": [ + { + "id": "custom.width", + "value": 90 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "empty", + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 42 + }, + "id": 43, + "interval": "1", + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 1, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "k6_http_reqs_total{testid=~\"$testid\"}", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0, sum by(name) (rate($metrics{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.5, sum by(name) (rate($metrics{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(name) (rate($metrics{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(1, sum by(name) (rate($metrics{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "B" + } + ], + "title": "Requests by URL", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["min", "max"], + "operation": "aggregate" + }, + "Value #A": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": [], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "method": { + "aggregations": [], + "operation": "groupby" + }, + "name": { + "aggregations": [], + "operation": "groupby" + }, + "scenario": { + "aggregations": [], + "operation": "groupby" + }, + "status": { + "aggregations": [], + "operation": "groupby" + }, + "testid": { + "aggregations": [], + "operation": "groupby" + }, + "tls_version": { + "aggregations": [] + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A (lastNotNull)": 5, + "Value #C (lastNotNull)": 6, + "Value #D (lastNotNull)": 7, + "method": 3, + "name": 1, + "scenario": 2, + "status": 4, + "testid": 0 + }, + "renameByName": { + "Value": "", + "Value #A (lastNotNull)": "count", + "Value #B (lastNotNull)": "max", + "Value #B (min)": "min", + "Value #C (lastNotNull)": "min ", + "Value #C (mean)": "", + "Value #D (lastNotNull)": "mean", + "Value #E (max)": "max", + "Value #F (lastNotNull)": "p95" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "count" + } + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 11, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 41, + "interval": "1s", + "options": { + "legend": { + "calcs": ["lastNotNull"], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "sortBy": "Last *", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.5.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_blocked_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_blocked_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Blocked - {{testid}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_connecting_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_connecting_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Connecting - {{testid}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_duration_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_duration_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Duration - {{testid}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_receiving_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_receiving_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Receiving - {{testid}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_sending_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_sending_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Sending - {{testid}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_tls_handshaking_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_tls_handshaking_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "TLS Handshaking - {{testid}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "avg by(testid)(histogram_sum(rate(k6_http_req_waiting_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_waiting_seconds{testid=~\"$testid\", name=~\"$url\",scenario=~\"$scenario\"}[$__rate_interval])))", + "hide": false, + "interval": "", + "legendFormat": "Waiting - {{testid}}", + "range": true, + "refId": "B" + } + ], + "title": "HTTP breakdown", + "transformations": [ + { + "id": "renameByRegex", + "options": { + "regex": "/(.* -) $/", + "renamePattern": "$1 without testid" + } + } + ], + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [ + { + "options": { + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Request Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95 Response Time(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RPS(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "empty", + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 54, + "interval": "5s", + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(testid)(rate($metrics{scenario=~\"$scenario\", expected_response=\"true\", testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) (max(k6_http_reqs_total{testid=~\"$testid\", scenario=~\"$scenario\"}))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(scenario) (sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) (k6_iterations_total{testid=~\"$testid\",scenario=~\"$scenario\"})", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) ( k6_http_reqs_total{testid=~\"$testid\",scenario=~\"$scenario\",expected_response=\"false\"})\n", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Scenarios", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #A": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [] + }, + "expected_response": { + "aggregations": [], + "operation": "groupby" + }, + "scenario": { + "aggregations": [], + "operation": "groupby" + }, + "test_type": { + "aggregations": [] + }, + "testid": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A (lastNotNull)": 5, + "Value #B (lastNotNull)": 4, + "Value #C (lastNotNull)": 6, + "Value #D (lastNotNull)": 2, + "Value #E (lastNotNull)": 3, + "scenario": 1, + "testid": 0 + }, + "renameByName": { + "Value #A (lastNotNull)": "P95 Response Time(AVG)", + "Value #B (lastNotNull)": "Failed Request Count", + "Value #C (lastNotNull)": "RPS(AVG)", + "Value #C (max)": "RPS(AVG)", + "Value #D (lastNotNull)": "Iterations", + "Value #E (lastNotNull)": "Request Count", + "Value (lastNotNull)": "Failed Request", + "scenario": "Scenario", + "testid": "TesID" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "Request Count" + }, + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "P95 Response Time(AVG)" + } + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "table" + }, + { + "collapsed": false, + "datasource": { + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 35, + "panels": [], + "repeat": "scenario", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "refId": "A" + } + ], + "title": "Scenario level metrics $testid $scenario", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Request Count" + }, + "properties": [ + { + "id": "unit", + "value": "none" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95 Response Time(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RPS(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "TestID" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "empty", + "result": { + "index": 0, + "text": "without testid" + } + }, + "type": "special" + } + ] + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 116, + "interval": "5s", + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "frameIndex": 1, + "showHeader": true + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by(testid)(rate($metrics{scenario=~\"$scenario\", expected_response=\"true\", testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) (max(k6_http_reqs_total{testid=~\"$testid\", scenario=~\"$scenario\"}))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "avg by(scenario) (sum(irate(k6_http_reqs_total{testid=~\"$testid\"}[$__rate_interval])))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) (k6_iterations_total{testid=~\"$testid\",scenario=~\"$scenario\"})", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by(scenario) ( k6_http_reqs_total{testid=~\"$testid\",scenario=~\"$scenario\",expected_response=\"false\"})\n", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + } + ], + "title": "Scenarios", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Value": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #A": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #F": { + "aggregations": [] + }, + "expected_response": { + "aggregations": [], + "operation": "groupby" + }, + "scenario": { + "aggregations": [], + "operation": "groupby" + }, + "test_type": { + "aggregations": [] + }, + "testid": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": { + "Value #A (lastNotNull)": 5, + "Value #B (lastNotNull)": 4, + "Value #C (lastNotNull)": 6, + "Value #D (lastNotNull)": 2, + "Value #E (lastNotNull)": 3, + "scenario": 1, + "testid": 0 + }, + "renameByName": { + "Value #A (lastNotNull)": "P95 Response Time(AVG)", + "Value #B (lastNotNull)": "Failed Request Count", + "Value #C (lastNotNull)": "RPS(AVG)", + "Value #C (max)": "RPS(AVG)", + "Value #D (lastNotNull)": "Iterations", + "Value #E (lastNotNull)": "Request Count", + "Value (lastNotNull)": "Failed Request", + "scenario": "Scenario", + "testid": "TestID" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "Request Count" + }, + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "P95 Response Time(AVG)" + } + ], + "match": "any", + "type": "exclude" + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Response Time (avg) - " + }, + "properties": [ + { + "id": "displayName", + "value": "Response Time (avg) - without testid" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "A" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "left" + }, + { + "id": "color", + "value": { + "mode": "fixed" + } + }, + { + "id": "custom.axisLabel", + "value": "VUs" + }, + { + "id": "custom.scaleDistribution", + "value": { + "log": 10, + "type": "log" + } + }, + { + "id": "custom.showPoints", + "value": "never" + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "B" + }, + "properties": [ + { + "id": "custom.lineStyle", + "value": { + "dash": [10, 10], + "fill": "dash" + } + }, + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + }, + { + "id": "custom.axisLabel", + "value": "RPS" + }, + { + "id": "custom.lineWidth", + "value": 3 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "D" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + }, + { + "id": "custom.lineWidth", + "value": 5 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.axisPlacement", + "value": "hidden" + }, + { + "id": "decimals", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + }, + { + "id": "unit", + "value": "s" + }, + { + "id": "custom.fillOpacity", + "value": 18 + }, + { + "id": "custom.lineInterpolation", + "value": "smooth" + }, + { + "id": "custom.axisLabel", + "value": "Response Time" + }, + { + "id": "decimals", + "value": 2 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Active VUs - " + }, + "properties": [ + { + "id": "displayName", + "value": "Active VUs - without testid" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Requests Rate - " + }, + "properties": [ + { + "id": "displayName", + "value": "Requests Rate - without testid" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed Requests Rate - " + }, + "properties": [ + { + "id": "displayName", + "value": "Failed Requests Rate - without testid" + } + ] + } + ] + }, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 77 + }, + "id": 117, + "interval": "5", + "options": { + "legend": { + "calcs": ["min", "mean", "max", "lastNotNull"], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "timezones": ["browser"], + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "k6_vus{testid=~\"$testid\"}", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Active VUs - {{testid}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by(testid) (rate(k6_http_reqs_total{testid=~\"$testid\",scenario=~\"$scenario\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "Requests Rate - {{testid}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "avg by(testid) (sum by(testid) (rate(k6_http_reqs_total{testid=~\"$testid\",scenario=~\"$scenario\", expected_response=\"false\"}[$__rate_interval])))", + "hide": false, + "instant": false, + "legendFormat": "Failed Requests Rate - {{testid}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(histogram_sum(rate(k6_http_req_duration_seconds{testid=~\"$testid\",scenario=~\"$scenario\"}[$__rate_interval]))\n/\nhistogram_count(rate(k6_http_req_duration_seconds{testid=~\"$testid\",scenario=~\"$scenario\"}[$__rate_interval]))) ", + "hide": false, + "legendFormat": "Response Time (avg) - {{testid}}", + "range": true, + "refId": "C" + } + ], + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": ["prometheus", "k6"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(testid)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "testid", + "options": [], + "query": { + "query": "label_values(testid)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 3, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(scenario)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "scenario", + "options": [], + "query": { + "query": "label_values(scenario)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": "http.*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(url)", + "hide": 0, + "includeAll": true, + "multi": false, + "name": "url", + "options": [], + "query": { + "query": "label_values(url)", + "refId": "StandardVariableQuery" + }, + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "filters": [], + "hide": 0, + "name": "Filters", + "skipUrlSync": false, + "type": "adhoc" + }, + { + "current": { + "selected": false, + "text": "k6_http_req_waiting_seconds", + "value": "k6_http_req_waiting_seconds" + }, + "description": "Metrics for apdex", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "metrics", + "options": [ + { + "selected": true, + "text": "k6_http_req_waiting_seconds", + "value": "k6_http_req_waiting_seconds" + }, + { + "selected": false, + "text": "k6_http_req_duration_seconds", + "value": "k6_http_req_duration_seconds" + } + ], + "query": "k6_http_req_waiting_seconds, k6_http_req_duration_seconds", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Test Result", + "uid": "01npcT44k", + "version": 8, + "weekStart": "" +} diff --git a/benchmarks/grafana/dashboards/dashboard-tests.json b/benchmarks/grafana/dashboards/dashboard-tests.json new file mode 100644 index 00000000..75ccb0a8 --- /dev/null +++ b/benchmarks/grafana/dashboards/dashboard-tests.json @@ -0,0 +1,518 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.3" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "table", + "name": "Table", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "List of k6 test runs", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": true, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "filterable": true, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time (min)" + }, + "properties": [ + { + "id": "displayName", + "value": "start" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + }, + { + "id": "custom.align" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "testid" + }, + "properties": [ + { + "id": "links", + "value": [ + { + "targetBlank": true, + "title": "", + "url": "http://localhost:3000/d/01npcT44k/prometheus-result?orgId=1&var-testid=${__value.raw}&from=${__data.fields['start'].numeric}&to=${__data.fields['end'].numeric}\n" + } + ] + }, + { + "id": "custom.width", + "value": 188 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time (max)" + }, + "properties": [ + { + "id": "displayName", + "value": "end" + }, + { + "id": "unit", + "value": "time: YYYY-MM-DD HH:mm:ss" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "start" + }, + "properties": [ + { + "id": "custom.width", + "value": 179 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "P95 Response Time(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "s" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RPS(AVG)" + }, + "properties": [ + { + "id": "unit", + "value": "reqps" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Failed Request Count" + }, + "properties": [ + { + "id": "mappings", + "value": [ + { + "options": { + "match": "null+nan", + "result": { + "color": "blue", + "index": 0, + "text": "0" + } + }, + "type": "special" + } + ] + }, + { + "id": "thresholds", + "value": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + { + "id": "color", + "value": { + "mode": "thresholds" + } + } + ] + } + ] + }, + "gridPos": { + "h": 19, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "interval": "5s", + "options": { + "footer": { + "countRows": false, + "fields": "", + "reducer": ["sum"], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "9.4.3", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "k6_http_reqs_total", + "format": "table", + "legendFormat": "__auto", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(k6_http_reqs_total{})", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(k6_http_req_duration_seconds{expected_response=\"true\"}[$__range])))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(irate(k6_http_reqs_total{}[$__range]))", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum (k6_http_reqs_total{expected_response=\"false\"})", + "format": "table", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "E" + } + ], + "title": "Test runs", + "transformations": [ + { + "id": "merge", + "options": {} + }, + { + "id": "groupBy", + "options": { + "fields": { + "Time": { + "aggregations": ["min", "max"], + "operation": "aggregate" + }, + "Value": { + "aggregations": [] + }, + "Value #B": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #C": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #D": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "Value #E": { + "aggregations": ["lastNotNull"], + "operation": "aggregate" + }, + "artifact": { + "aggregations": [], + "operation": "groupby" + }, + "branch": { + "aggregations": [], + "operation": "groupby" + }, + "build": { + "aggregations": [], + "operation": "groupby" + }, + "commit": { + "aggregations": [], + "operation": "groupby" + }, + "team": { + "aggregations": [], + "operation": "groupby" + }, + "testFileName": { + "aggregations": [], + "operation": "groupby" + }, + "testid": { + "aggregations": [], + "operation": "groupby" + }, + "typeArtifact": { + "aggregations": [], + "operation": "groupby" + }, + "version": { + "aggregations": [], + "operation": "groupby" + } + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "", + "binary": { + "left": "Time (min)", + "operator": "-", + "reducer": "sum", + "right": "60000" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": "Time (max)", + "operator": "+", + "reducer": "sum", + "right": "60000" + }, + "mode": "binary", + "reduce": { + "include": ["Time (max)"], + "reducer": "sum" + } + } + }, + { + "id": "organize", + "options": { + "excludeByName": { + "Time (max)": true, + "Time (max) + 60": true, + "Time (min)": true, + "Time (min) - 60": true + }, + "indexByName": { + "Time (max)": 3, + "Time (max) + 60000": 4, + "Time (min)": 2, + "Time (min) - 60000": 1, + "Value #B (lastNotNull)": 5, + "Value #C (lastNotNull)": 6, + "Value #D (lastNotNull)": 7, + "testid": 0 + }, + "renameByName": { + "Time (max) + 10000": "Time (max)", + "Time (max) + 60": "Time (max)", + "Time (max) + 60000": "Time (max)", + "Time (min) - 60": "Time (min)", + "Time (min) - 60000": "Time (min)", + "Time (min) 2": "", + "Value #B (lastNotNull)": "Request Count", + "Value #C (lastNotNull)": "P95 Response Time(AVG)", + "Value #D (lastNotNull)": "RPS(AVG)", + "Value #E (lastNotNull)": "Failed Request Count", + "testid": "" + } + } + }, + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "isNull", + "options": {} + }, + "fieldName": "Request Count" + } + ], + "match": "all", + "type": "exclude" + } + } + ], + "type": "table" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": ["prometheus", "k6"], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "DS_PROMETHEUS", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Tests List", + "uid": "UoYDJHJhk", + "version": 4, + "weekStart": "" +} diff --git a/benchmarks/grafana/dashboards/dashboards.yaml b/benchmarks/grafana/dashboards/dashboards.yaml new file mode 100644 index 00000000..8b705585 --- /dev/null +++ b/benchmarks/grafana/dashboards/dashboards.yaml @@ -0,0 +1,14 @@ +# For configuration options, see +# https://grafana.com/docs/grafana/latest/administration/provisioning/#dashboards + +apiVersion: 1 + +providers: + # We're defining a directory from which to load file-based dashboards + - name: "prometheus" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + editable: true + options: + path: /etc/grafana/provisioning/dashboards diff --git a/benchmarks/grafana/datasources/datasource.yaml b/benchmarks/grafana/datasources/datasource.yaml new file mode 100644 index 00000000..06a635d8 --- /dev/null +++ b/benchmarks/grafana/datasources/datasource.yaml @@ -0,0 +1,17 @@ +# For configuration options, see +# https://grafana.com/docs/grafana/latest/administration/provisioning/#example-data-source-config-file + +apiVersion: 1 + +datasources: + - name: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: false + - name: jaeger + type: jaeger + access: proxy + url: http://jaeger:16686 + editable: false