From 872e7243ec23f4fc9a6789a977e14653d2f00b6b Mon Sep 17 00:00:00 2001 From: Olaf Lessenich Date: Wed, 30 Oct 2024 23:23:51 +0100 Subject: [PATCH] feat: export smoke test results in prometheus format In addition to the html report, the smoke test results can now be exported in prometheus format. The exporter is implemented as a custom Playwright reporter that writes the results to a file and optionally sends them to a Prometheus Pushgateway. The configuration for the Prometheus Pushgateway is read from environment variables to enable easy integration with CI/CD using secrets. This should ensure easy integration of the smoke tests into existing monitoring solutions by either configuring the Prometheus Pushgateway in the CI workflow or by scraping the results file from the Github Actions artifacts. Additionally, we enable the json reporter for the smoke tests to provide more easily parseable results for further processing if needed. All reporters are now configured to save their output to the `webui//playwright-report` directory to easily distinguish them from the test result artifacts, such as screenshots and videos, that are saved in the `webui/test-results` directory. Contributed on behalf of STMicroelectronics Signed-off-by: Olaf Lessenich --- webui/configs/playwright.config.ts | 17 +++++- webui/package.json | 1 + webui/reporters/prometheus-reporter.ts | 80 ++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 webui/reporters/prometheus-reporter.ts diff --git a/webui/configs/playwright.config.ts b/webui/configs/playwright.config.ts index 7ec00eb9f..4d681c90a 100644 --- a/webui/configs/playwright.config.ts +++ b/webui/configs/playwright.config.ts @@ -1,5 +1,8 @@ import { defineConfig, devices } from "@playwright/test"; import { SmokeTestOptions } from "./smoke-test.options"; +import path from "path"; + +const resultsDir = path.resolve('./', 'playwright-report') /** * See https://playwright.dev/docs/test-configuration. @@ -14,8 +17,20 @@ export default defineConfig({ retries: process.env.CI ? 2 : 0, /* Opt out of parallel tests on CI. */ workers: process.env.CI ? 1 : undefined, + + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ - reporter: "html", + reporter: [ + [ + "html", { outputFolder: `${resultsDir}/playwright-report` } + ], + [ + "json", { outputFile: `${resultsDir}/report.json` } + ], + [ + "../reporters/prometheus-reporter.ts", { outputFile: `${resultsDir}/report.prom` } + ] + ], /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ use: { /* Base URL to use in actions like `await page.goto('/')`. */ diff --git a/webui/package.json b/webui/package.json index d7ed3c66c..f5b680008 100644 --- a/webui/package.json +++ b/webui/package.json @@ -86,6 +86,7 @@ "@types/react-transition-group": "^4.4.6", "@typescript-eslint/eslint-plugin": "^5.61.0", "@typescript-eslint/parser": "^5.61.0", + "axios": "^1.7.7", "chai": "^4.3.7", "css-loader": "^6.8.1", "eslint": "^8.44.0", diff --git a/webui/reporters/prometheus-reporter.ts b/webui/reporters/prometheus-reporter.ts new file mode 100644 index 000000000..f4aba5ffa --- /dev/null +++ b/webui/reporters/prometheus-reporter.ts @@ -0,0 +1,80 @@ +import { Reporter, TestResult } from "@playwright/test/reporter"; +import fs from "fs"; +import axios from "axios"; + +type PrometheusReporterOptions = { + outputFile: string; +}; + +class PrometheusReporter implements Reporter { + private pushGatewayUrl: string; + private username: string; + private password: string; + private outputFile: string; + private totalTests: number; + private passedTests: number; + private failedTests: number; + private skippedTests: number; + private startTime: number; + private duration: number; + + constructor(options: PrometheusReporterOptions) { + this.pushGatewayUrl = process.env.PROMETHEUS_PUSHGATEWAY_URL || ""; + this.username = process.env.PROMETHEUS_USERNAME || ""; + this.password = process.env.PROMETHEUS_PASSWORD || ""; + this.outputFile = options.outputFile; + this.totalTests = 0; + this.passedTests = 0; + this.failedTests = 0; + this.skippedTests = 0; + this.startTime = Date.now(); + } + + onTestEnd(_: unknown, result: TestResult) { + this.totalTests++; + if (result.status === "passed") this.passedTests++; + if (result.status === "failed") this.failedTests++; + if (result.status === "skipped") this.skippedTests++; + } + + async onEnd() { + this.duration = Date.now() - this.startTime; + const content = this.buildMetricsString(); + fs.writeFileSync(this.outputFile, content); + this.pushMetricsToPrometheus(content); + } + + private pushMetricsToPrometheus(content: string) { + if (!this.pushGatewayUrl) return; + axios.post(this.pushGatewayUrl, content, { + headers: { + "Content-Type": "text/plain", + }, + auth: { + username: this.username, + password: this.password, + }, + }); + } + + private buildMetricsString() { + return `# HELP playwright_test_total The total number of tests +# TYPE playwright_test_total counter +playwright_test_total ${this.totalTests} +# HELP playwright_test_passed The number of passed tests +# TYPE playwright_test_passed counter +playwright_test_passed ${this.passedTests} +# HELP playwright_test_failed The number of failed tests +# TYPE playwright_test_failed counter +playwright_test_failed ${this.failedTests} +# HELP playwright_test_skipped The number of skipped tests +# TYPE playwright_test_skipped counter +playwright_test_skipped ${this.skippedTests} +# HELP playwright_test_duration The duration of the test run +# TYPE playwright_test_duration gauge +playwright_test_duration ${this.duration} +`; + } +} + +export default PrometheusReporter;