WIP copy ndc-postgres benchmarks

benchmarks/component/README.md

@@ -0,0 +1,46 @@
+# Component benchmarks
+
+This is the benchmark suite for the SQLServer data connector.
+
+Running `run.sh` with a benchmark name as an argument will:
+
+1. build the SQLServer data connector Docker image,
+2. start the database with Chinook data,
+3. start the agent using an associated deployment, and
+4. run a benchmark using k6.
+
+Running without arguments will list available benchmarks.
+
+Everything is run through Docker Compose.
+
+The Docker image for the agent is built with Nix. If you haven't built with Nix
+before (or it's been a while), this may take some time at first.
+
+## Requirements
+
+1. _Nix_, to build the Docker image
+   1. Install [Nix](https://nixos.org/download.html)
+   2. Configure Nix by adding the following line to `~/.config/nix/nix.conf`:
+      ```
+      extra-experimental-features = flakes nix-command
+      ```
+2. _Docker_ and _Docker Compose_, to run the containers (see the root README)
+
+## Viewing the benchmark results
+
+When the benchmarks finish, the results will be printed.
+
+There is a Grafana dashboard which can be viewed as follows:
+
+1. Open [http://localhost:64300][].
+2. Open the menu on the left and choose "Dashboards".
+3. Choose the "Test Result" dashboard.
+
+## Adding a benchmark
+
+You can add a benchmark by copying one of the files in the "benchmarks"
+subdirectory and altering it.
+
+Please make sure that the name of the file corresponds to the `testid`.
+
+For further information, consult the [k6 documentation](https://k6.io/docs/).

benchmarks/component/benchmarks/select-by-pk.js

@@ -0,0 +1,67 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-by-pk";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      id: { type: "column", column: "AlbumId", arguments: {} },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "AlbumId",
+        path: [],
+      },
+      operator: {
+        type: "equal",
+      },
+      value: {
+        type: "scalar",
+        value: 1,
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/benchmarks/select-variables.js

@@ -0,0 +1,89 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-variables";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      Title: {
+        type: "column",
+        column: "Title",
+        arguments: {},
+      },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "Title",
+        path: [],
+      },
+      operator: {
+        type: "other",
+        name: "_like",
+      },
+      value: {
+        type: "variable",
+        name: "search",
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+  variables: [
+    {
+      search: "%Garage%",
+    },
+    {
+      search: "%Good%",
+    },
+    {
+      search: "%Rock%",
+    },
+    {
+      search: "%Dog%",
+    },
+    {
+      search: "%Log%",
+    },
+  ],
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/benchmarks/select-where.js

@@ -0,0 +1,70 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select-where";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      id: { type: "column", column: "AlbumId", arguments: {} },
+      title: { type: "column", column: "Title", arguments: {} },
+      artist_id: { type: "column", column: "ArtistId", arguments: {} },
+    },
+    where: {
+      type: "binary_comparison_operator",
+      column: {
+        type: "column",
+        name: "Title",
+        path: [],
+      },
+      operator: {
+        type: "other",
+        name: "_ilike",
+      },
+      value: {
+        type: "scalar",
+        value: "%a%",
+      },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/benchmarks/select.js

@@ -0,0 +1,54 @@
+import { check } from "k6";
+import http from "k6/http";
+import { newSummaryHandler } from "../common.js";
+
+const testid = "select";
+const agentSocket = __ENV.AGENT_SOCKET || "localhost:8100";
+const url = `http://${agentSocket}/query`;
+const data = {
+  collection: "Album",
+  query: {
+    fields: {
+      id: { type: "column", column: "AlbumId", arguments: {} },
+      title: { type: "column", column: "Title", arguments: {} },
+      artist_id: { type: "column", column: "ArtistId", arguments: {} },
+    },
+  },
+  arguments: {},
+  collection_relationships: {},
+};
+
+export default function () {
+  const response = http.post(url, JSON.stringify(data), {
+    headers: {
+      "Content-Type": "application/json",
+    },
+  });
+
+  check(response, {
+    "status is 200": (r) => r.status == 200,
+  });
+}
+
+export const handleSummary = newSummaryHandler(testid);
+
+export const options = {
+  tags: {
+    testid,
+  },
+  scenarios: {
+    short_sustained: {
+      executor: "constant-vus",
+      vus: 100,
+      duration: "10s",
+    },
+  },
+  thresholds: {
+    checks: [
+      {
+        threshold: "rate == 1",
+        abortOnFail: true,
+      },
+    ],
+  },
+};

benchmarks/component/common.js

@@ -0,0 +1,21 @@
+// Sadly, k6 does not export its default text summarizer directly,
+// instead making it available as a separate library.
+// We need to import it from there, via their CDN.
+import { textSummary } from "https://jslib.k6.io/k6-summary/0.0.3/index.js";
+
+// This constructs a summary handler, injecting the test ID.
+export function newSummaryHandler(testid) {
+  return function handleSummary(data) {
+    const summaries = {
+      // We always want to print a text summary.
+      stdout: textSummary(data, { indent: " ", enableColors: true }),
+    };
+    const outputDirectory = __ENV.OUTPUT_DIRECTORY;
+    if (outputDirectory) {
+      const summaryFile = `${outputDirectory}/summaries/${testid}__${new Date().toISOString()}.json`;
+      // If there is an output directory provided, we also serialize the summary to JSON and write it there.
+      summaries[summaryFile] = JSON.stringify(data);
+    }
+    return summaries;
+  };
+}