Enable benchmarks

.github/workflows/benchmarks.yaml

@@ -0,0 +1,92 @@
+on: [push]
+
+name: SQLServer NDC component benchmarks
+
+permissions:
+  contents: write
+  deployments: write
+
+jobs:
+  benchmark:
+    name: Benchmark
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout 🛎️
+        uses: actions/checkout@v4
+
+      - name: Install Nix ❄
+        uses: cachix/install-nix-action@v23
+        with:
+          github_access_token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up the Nix Cache 🔌
+        uses: cachix/cachix-action@v12
+        with:
+          name: hasura-v3-dev
+          authToken: ${{ secrets.CACHIX_AUTH_TOKEN }}
+
+      - name: Build the Docker image 🔨
+        run: |
+          docker load < $(nix build --no-link --print-out-paths '.#docker')
+
+      - name: Start dependencies ▶️
+        run: |
+          cd benchmarks/component
+          docker compose up --detach --wait sqlserver grafana
+
+      - name: Generate the deployment configuration 🚧
+        run: |
+          set -e -u -o pipefail
+          cd benchmarks/component
+          mkdir -p generated
+          docker compose up --detach --wait agent-configuration
+          CONFIGURATION_SERVER_PORT="$(docker compose port agent-configuration 9100 | sed 's/.\+://')"
+          CONFIGURATION_SERVER="localhost:${CONFIGURATION_SERVER_PORT}"
+          SQLSERVER_CONNECTION_STRING="DRIVER={ODBC Driver 18 for SQL Server};SERVER=sqlserver,1433;Uid=SA;Database=Chinook;Pwd=Password!"
+          ../../scripts/new-configuration.sh "$CONFIGURATION_SERVER" "${SQLSERVER_CONNECTION_STRING}" \
+            | tee ./generated/deployment.json
+          docker compose down agent-configuration
+
+      - name: Run benchmarks 🏃
+        run: |
+          cd benchmarks/component
+          for benchmark in $(ls benchmarks); do
+            echo "Starting agent..."
+            docker compose up --wait agent
+            echo "Running ${benchmark}..."
+            docker compose run --rm benchmark run "/benchmarks/$benchmark"
+            echo "Saving metrics for ${benchmark}..."
+            ./metrics.sh localhost:8100 ${benchmark}
+            echo "Stopping agent..."
+            docker compose down agent
+          done
+
+      - name: Extract summaries ⛏️
+        run: |
+          ./benchmarks/component/summarize.sh | tee benchmark-results.json
+
+      # We run these benchmarks on every push so we can easily see the results.
+      # However, we only store the results on `main`.
+      - name: Store benchmark result ⬆️
+        if: github.ref == 'refs/heads/main'
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Component benchmarks
+          tool: customSmallerIsBetter
+          output-file-path: benchmark-results.json
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: true
+          alert-threshold: 200%
+          comment-on-alert: true
+
+      # scream into Slack if something goes wrong
+      - name: Report Status
+        if: always() && github.ref == 'refs/heads/main'
+        uses: ravsamhq/notify-slack-action@v2
+        with:
+          status: ${{ job.status }}
+          notify_when: failure
+          notification_title: "😧 Error on <{repo_url}|{repo}>"
+          message_format: "🐴 *{workflow}* {status_message} for <{repo_url}|{repo}>"
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.BROKEN_BUILD_SLACK_WEBHOOK_URL }}

benchmarks/component/README.md

@@ -0,0 +1,46 @@
+# Component benchmarks
+
+This is the benchmark suite for the SQLServer data connector.
+
+Running `run.sh` with a benchmark name as an argument will:
+
+1. build the SQLServer data connector Docker image,
+2. start the database with Chinook data,
+3. start the agent using an associated deployment, and
+4. run a benchmark using k6.
+
+Running without arguments will list available benchmarks.
+
+Everything is run through Docker Compose.
+
+The Docker image for the agent is built with Nix. If you haven't built with Nix
+before (or it's been a while), this may take some time at first.
+
+## Requirements
+
+1. _Nix_, to build the Docker image
+   1. Install [Nix](https://nixos.org/download.html)
+   2. Configure Nix by adding the following line to `~/.config/nix/nix.conf`:
+      ```
+      extra-experimental-features = flakes nix-command
+      ```
+2. _Docker_ and _Docker Compose_, to run the containers (see the root README)
+
+## Viewing the benchmark results
+
+When the benchmarks finish, the results will be printed.
+
+There is a Grafana dashboard which can be viewed as follows:
+
+1. Open [http://localhost:64300][].
+2. Open the menu on the left and choose "Dashboards".
+3. Choose the "Test Result" dashboard.
+
+## Adding a benchmark
+
+You can add a benchmark by copying one of the files in the "benchmarks"
+subdirectory and altering it.
+
+Please make sure that the name of the file corresponds to the `testid`.
+
+For further information, consult the [k6 documentation](https://k6.io/docs/).

benchmarks/component/benchmarks/select-by-pk.js

@@ -0,0 +1,67 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-by-pk";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      id: { type: "column", column: "AlbumId", arguments: {} },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "AlbumId",
+        path: [],
+      },
+      operator: {
+        type: "equal",
+      },
+      value: {
+        type: "scalar",
+        value: 1,
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/benchmarks/select-variables.js

@@ -0,0 +1,89 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-variables";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      Title: {
+        type: "column",
+        column: "Title",
+        arguments: {},
+      },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "Title",
+        path: [],
+      },
+      operator: {
+        type: "other",
+        name: "_like",
+      },
+      value: {
+        type: "variable",
+        name: "search",
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+  variables: [
+    {
+      search: "%Garage%",
+    },
+    {
+      search: "%Good%",
+    },
+    {
+      search: "%Rock%",
+    },
+    {
+      search: "%Dog%",
+    },
+    {
+      search: "%Log%",
+    },
+  ],
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/benchmarks/select-where.js

@@ -0,0 +1,70 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-where";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      id: { type: "column", column: "AlbumId", arguments: {} },
+      title: { type: "column", column: "Title", arguments: {} },
+      artist_id: { type: "column", column: "ArtistId", arguments: {} },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "Title",
+        path: [],
+      },
+      operator: {
+        type: "other",
+        name: "_like",
+      },
+      value: {
+        type: "scalar",
+        value: "%a%",
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};