feat: Add n8n-benchmark cli (no-changelog) (#10410)

.github/workflows/docker-images-benchmark.yml

@@ -0,0 +1,43 @@
+name: Benchmark Docker Image CI
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - 'packages/benchmark/**'
+      - 'pnpm-lock.yaml'
+      - 'pnpm-workspace.yaml'
+      - '.github/workflows/docker-images-benchmark.yml'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/[email protected]
+
+      - name: Set up QEMU
+        uses: docker/[email protected]
+
+      - name: Set up Docker Buildx
+        uses: docker/[email protected]
+
+      - name: Login to GitHub Container Registry
+        uses: docker/[email protected]
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build
+        uses: docker/[email protected]
+        with:
+          context: .
+          file: ./packages/benchmark/Dockerfile
+          platforms: linux/amd64
+          provenance: false
+          push: true
+          tags: |
+            ghcr.io/${{ github.repository_owner }}/n8n-benchmark:latest

packages/@n8n/benchmark/Dockerfile

@@ -0,0 +1,62 @@
+# syntax=docker/dockerfile:1
+FROM node:20.16.0 AS base
+
+# Install required dependencies
+RUN apt-get update && apt-get install -y gnupg2 curl
+
+# Add k6 GPG key and repository
+RUN mkdir -p /etc/apt/keyrings && \
+	curl -sS https://dl.k6.io/key.gpg | gpg --dearmor --yes -o /etc/apt/keyrings/k6.gpg && \
+	chmod a+x /etc/apt/keyrings/k6.gpg && \
+	echo "deb [signed-by=/etc/apt/keyrings/k6.gpg] https://dl.k6.io/deb stable main" | tee /etc/apt/sources.list.d/k6.list
+
+# Update and install k6
+RUN apt-get update && \
+	apt-get install -y k6 tini && \
+	apt-get clean && \
+	rm -rf /var/lib/apt/lists/*
+
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+RUN corepack enable
+
+#
+# Builder
+FROM base AS builder
+
+WORKDIR /app
+
+COPY --chown=node:node ./pnpm-lock.yaml /app/pnpm-lock.yaml
+COPY --chown=node:node ./pnpm-workspace.yaml /app/pnpm-workspace.yaml
+COPY --chown=node:node ./package.json /app/package.json
+COPY --chown=node:node ./packages/@n8n/benchmark/package.json /app/packages/@n8n/benchmark/package.json
+COPY --chown=node:node ./patches /app/patches
+COPY --chown=node:node ./scripts /app/scripts
+
+RUN pnpm install --frozen-lockfile
+
+# TS config files
+COPY --chown=node:node ./tsconfig.json /app/tsconfig.json
+COPY --chown=node:node ./tsconfig.build.json /app/tsconfig.build.json
+COPY --chown=node:node ./tsconfig.backend.json /app/tsconfig.backend.json
+COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.json /app/packages/@n8n/benchmark/tsconfig.json
+COPY --chown=node:node ./packages/@n8n/benchmark/tsconfig.build.json /app/packages/@n8n/benchmark/tsconfig.build.json
+
+# Source files
+COPY --chown=node:node ./packages/@n8n/benchmark/src /app/packages/@n8n/benchmark/src
+COPY --chown=node:node ./packages/@n8n/benchmark/bin /app/packages/@n8n/benchmark/bin
+COPY --chown=node:node ./packages/@n8n/benchmark/scenarios /app/packages/@n8n/benchmark/scenarios
+
+WORKDIR /app/packages/@n8n/benchmark
+RUN pnpm build
+
+#
+# Runner
+FROM base AS runner
+
+COPY --from=builder /app /app
+
+WORKDIR /app/packages/@n8n/benchmark
+USER node
+
+ENTRYPOINT [ "/app/packages/@n8n/benchmark/bin/n8n-benchmark" ]

packages/@n8n/benchmark/README.md

@@ -0,0 +1,55 @@
+# n8n benchmarking tool
+
+Tool for executing benchmarks against an n8n instance.
+
+## Running locally with Docker
+
+Build the Docker image:
+
+```sh
+# Must be run in the repository root
+# k6 doesn't have an arm64 build available for linux, we need to build against amd64
+docker build --platform linux/amd64 -t n8n-benchmark -f packages/@n8n/benchmark/Dockerfile .
+```
+
+Run the image
+
+```sh
+docker run \
+  -e [email protected] \
+  -e N8N_USER_PASSWORD=password \
+  # For macos, n8n running outside docker
+  -e N8N_BASE_URL=http://host.docker.internal:5678 \
+  n8n-benchmark
+```
+
+## Running locally without Docker
+
+Requirements:
+
+- [k6](https://grafana.com/docs/k6/latest/set-up/install-k6/)
+- Node.js v20 or higher
+
+```sh
+pnpm build
+
+# Run tests against http://localhost:5678 with specified email and password
+[email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+
+# If you installed k6 using brew, you might have to specify it explicitly
+K6_PATH=/opt/homebrew/bin/k6 [email protected] N8N_USER_PASSWORD=password ./bin/n8n-benchmark run
+```
+
+## Configuration
+
+The configuration options the cli accepts can be seen from [config.ts](./src/config/config.ts)
+
+## Benchmark scenarios
+
+A benchmark scenario defines one or multiple steps to execute and measure. It consists of:
+
+- Manifest file which describes and configures the scenario
+- Any test data that is imported before the scenario is run
+- A [`k6`](https://grafana.com/docs/k6/latest/using-k6/http-requests/) script which executes the steps and receives `API_BASE_URL` environment variable in runtime.
+
+Available scenarios are located in [`./scenarios`](./scenarios/).

packages/@n8n/benchmark/bin/n8n-benchmark

@@ -0,0 +1,13 @@
+#!/usr/bin/env node
+
+// Check if version should be displayed
+const versionFlags = ['-v', '-V', '--version'];
+if (versionFlags.includes(process.argv.slice(-1)[0])) {
+	console.log(require('../package').version);
+	process.exit(0);
+}
+
+(async () => {
+	const oclif = require('@oclif/core');
+	await oclif.execute({ dir: __dirname });
+})();

packages/@n8n/benchmark/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@n8n/n8n-benchmark",
+  "version": "1.0.0",
+  "description": "Cli for running benchmark tests for n8n",
+  "main": "dist/index",
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json && tsc-alias -p tsconfig.build.json",
+    "start": "./bin/n8n-benchmark",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "typecheck": "tsc --noEmit",
+    "watch": "concurrently \"tsc -w -p tsconfig.build.json\" \"tsc-alias -w -p tsconfig.build.json\""
+  },
+  "engines": {
+    "node": ">=20.10"
+  },
+  "keywords": [
+    "automate",
+    "automation",
+    "IaaS",
+    "iPaaS",
+    "n8n",
+    "workflow",
+    "benchmark",
+    "performance"
+  ],
+  "dependencies": {
+    "@oclif/core": "4.0.7",
+    "axios": "catalog:",
+    "convict": "6.2.4",
+    "dotenv": "8.6.0",
+    "zx": "^8.1.4"
+  },
+  "devDependencies": {
+    "@types/convict": "^6.1.1",
+    "@types/k6": "^0.52.0",
+    "@types/node": "^20.14.8",
+    "tsc-alias": "^1.8.7",
+    "typescript": "^5.5.2"
+  },
+  "bin": {
+    "n8n-benchmark": "./bin/n8n-benchmark"
+  },
+  "oclif": {
+    "bin": "n8n-benchmark",
+    "commands": "./dist/commands",
+    "topicSeparator": " "
+  }
+}

packages/@n8n/benchmark/scenarios/scenario.schema.json

@@ -0,0 +1,42 @@
+{
+	"definitions": {
+		"ScenarioData": {
+			"type": "object",
+			"properties": {
+				"workflowFiles": {
+					"type": "array",
+					"items": {
+						"type": "string"
+					}
+				}
+			},
+			"required": [],
+			"additionalProperties": false
+		}
+	},
+	"type": "object",
+	"properties": {
+		"$schema": {
+			"type": "string",
+			"description": "The JSON schema to validate this file"
+		},
+		"name": {
+			"type": "string",
+			"description": "The name of the scenario"
+		},
+		"description": {
+			"type": "string",
+			"description": "A longer description of the scenario"
+		},
+		"scriptPath": {
+			"type": "string",
+			"description": "Relative path to the k6 test script"
+		},
+		"scenarioData": {
+			"$ref": "#/definitions/ScenarioData",
+			"description": "Data to import before running the scenario"
+		}
+	},
+	"required": ["name", "description", "scriptPath", "scenarioData"],
+	"additionalProperties": false
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.json

@@ -0,0 +1,25 @@
+{
+	"createdAt": "2024-08-06T12:19:51.268Z",
+	"updatedAt": "2024-08-06T12:20:45.000Z",
+	"name": "Single Webhook",
+	"active": true,
+	"nodes": [
+		{
+			"parameters": { "path": "single-webhook", "options": {} },
+			"id": "7587ab0e-cc15-424f-83c0-c887a0eb97fb",
+			"name": "Webhook",
+			"type": "n8n-nodes-base.webhook",
+			"typeVersion": 2,
+			"position": [760, 400],
+			"webhookId": "fa563fc2-c73f-4631-99a1-39c16f1f858f"
+		}
+	],
+	"connections": {},
+	"settings": { "executionOrder": "v1" },
+	"staticData": null,
+	"meta": { "templateCredsSetupCompleted": true, "responseMode": "lastNode", "options": {} },
+	"pinData": {},
+	"versionId": "840a38a1-ba37-433d-9f20-de73f5131a2b",
+	"triggerCount": 1,
+	"tags": []
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.manifest.json

@@ -0,0 +1,7 @@
+{
+	"$schema": "../scenario.schema.json",
+	"name": "SingleWebhook",
+	"description": "A single webhook trigger that responds with a 200 status code",
+	"scenarioData": { "workflowFiles": ["singleWebhook.json"] },
+	"scriptPath": "singleWebhook.script.ts"
+}

packages/@n8n/benchmark/scenarios/singleWebhook/singleWebhook.script.ts

@@ -0,0 +1,11 @@
+import http from 'k6/http';
+import { check } from 'k6';
+
+const apiBaseUrl = __ENV.API_BASE_URL;
+
+export default function () {
+	const res = http.get(`${apiBaseUrl}/webhook/single-webhook`);
+	check(res, {
+		'is status 200': (r) => r.status === 200,
+	});
+}

packages/@n8n/benchmark/src/commands/list.ts

@@ -0,0 +1,21 @@
+import { Command } from '@oclif/core';
+import { ScenarioLoader } from '@/scenario/scenarioLoader';
+import { loadConfig } from '@/config/config';
+
+export default class ListCommand extends Command {
+	static description = 'List all available scenarios';
+
+	async run() {
+		const config = loadConfig();
+		const scenarioLoader = new ScenarioLoader();
+
+		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+
+		console.log('Available test scenarios:');
+		console.log('');
+
+		for (const scenario of allScenarios) {
+			console.log('\t', scenario.name, ':', scenario.description);
+		}
+	}
+}

packages/@n8n/benchmark/src/commands/run.ts

@@ -0,0 +1,39 @@
+import { Command, Flags } from '@oclif/core';
+import { loadConfig } from '@/config/config';
+import { ScenarioLoader } from '@/scenario/scenarioLoader';
+import { ScenarioRunner } from '@/testExecution/scenarioRunner';
+import { N8nApiClient } from '@/n8nApiClient/n8nApiClient';
+import { ScenarioDataFileLoader } from '@/scenario/scenarioDataLoader';
+import { K6Executor } from '@/testExecution/k6Executor';
+
+export default class RunCommand extends Command {
+	static description = 'Run all (default) or specified test scenarios';
+
+	// TODO: Add support for filtering scenarios
+	static flags = {
+		scenarios: Flags.string({
+			char: 't',
+			description: 'Comma-separated list of test scenarios to run',
+			required: false,
+		}),
+	};
+
+	async run() {
+		const config = loadConfig();
+		const scenarioLoader = new ScenarioLoader();
+
+		const scenarioRunner = new ScenarioRunner(
+			new N8nApiClient(config.get('n8n.baseUrl')),
+			new ScenarioDataFileLoader(),
+			new K6Executor(config.get('k6ExecutablePath'), config.get('n8n.baseUrl')),
+			{
+				email: config.get('n8n.user.email'),
+				password: config.get('n8n.user.password'),
+			},
+		);
+
+		const allScenarios = scenarioLoader.loadAll(config.get('testScenariosPath'));
+
+		await scenarioRunner.runManyScenarios(allScenarios);
+	}
+}