feat: add flaky method

.github/workflows/main.yaml

@@ -20,3 +20,5 @@ jobs:
       run: npx tsc
     - name: Merge
       run: npx ctrf merge test-reports
+    - name: Flaky
+      run: npx ctrf flaky test-reports/ctrf-report-one.json

README.md

@@ -29,6 +29,8 @@ The only way we can grow CTRF is with your help and the support of the software
 | Name         |Details                                                                              |
 | ------------ | ----------------------------------------------------------------------------------- |
 | `merge`      | Merge multiple CTRF reports into a single report.                                   |
+| `flaky`      | Output flaky test name and retries.                                                 |
+
 
 ## Merge
 
@@ -42,7 +44,7 @@ npx ctrf merge <directory>
 
 Replace `directory` with the path to the directory containing the CTRF reports you want to merge.
 
-## Options
+### Options
 
 -o, --output `filename`: Output file name for the merged report. Default is ctrf-report.json.
 
@@ -62,12 +64,27 @@ npx ctrf merge <directory> --output-dir /path/to/output
 npx ctrf merge <directory> --keep-reports
 ```
 
-## Example
+## Flaky
+
+The flaky command is useful for identifying tests marked as flaky in your CTRF report. Flaky tests are tests that pass or fail inconsistently and may require special attention or retries to determine their reliability.
 
-Merge all CTRF reports in the ctrf directory into a single report named merged-report.json and save it in the output directory, keeping the original reports:
+Usage
+To output flaky tests, use the following command:
 
 ```sh
-npx ctrf merge ctrf --output merged-report.json --output-dir output --keep-reports
+npx ctrf flaky <file-path>
+```
+
+Replace <file-path> with the path to the CTRF report file you want to analyze.
+
+### Output
+
+The command will output the names of the flaky tests and the number of retries each test has undergone. For example:
+
+```zsh
+Processing report: reports/sample-report.json
+Found 1 flaky test(s) in reports/sample-report.json:
+- Test Name: Test 1, Retries: 2
 ```
 
 ## What is CTRF?

package.json

@@ -1,6 +1,6 @@
 {
   "name": "ctrf",
-  "version": "0.0.6",
+  "version": "0.0.9",
   "description": "",
   "main": "index.js",
   "scripts": {

src/cli.ts

@@ -1,9 +1,9 @@
 #!/usr/bin/env node
 
-import fs from 'fs';
-import path from 'path';
 import yargs from 'yargs/yargs';
 import { hideBin } from 'yargs/helpers';
+import { mergeReports } from './merge';
+import { identifyFlakyTests } from './flaky';
 
 const argv = yargs(hideBin(process.argv))
   .command(
@@ -35,90 +35,22 @@ const argv = yargs(hideBin(process.argv))
         });
     },
     async (argv) => {
-      try {
-        if (!argv.directory) {
-          throw new Error('The "directory" argument is required.');
-        }
-
-        const directoryPath = path.resolve(argv.directory as string);
-        const outputFileName = argv.output as string;
-        const outputDir = argv['output-dir'] ? path.resolve(argv['output-dir'] as string) : directoryPath;
-        const keepReports = argv['keep-reports'] as boolean;
-        const outputPath = path.join(outputDir, outputFileName);
-
-        console.log("Merging CTRF reports...");
-
-        const files = fs.readdirSync(directoryPath);
-
-        files.forEach((file) => {
-          console.log('Found CTRF report file:', file);
-        });
-
-        const ctrfReportFiles = files.filter((file) => {
-          try {
-            const filePath = path.join(directoryPath, file);
-            const fileContent = fs.readFileSync(filePath, 'utf8');
-            const jsonData = JSON.parse(fileContent);
-            return jsonData.hasOwnProperty('results');
-          } catch (error) {
-            console.error(`Error reading JSON file '${file}':`, error);
-            return false;
-          }
+      await mergeReports(argv.directory as string, argv.output as string, argv['output-dir'] as string, argv['keep-reports'] as boolean);
+    }
+  )
+  .command(
+    'flaky <file>',
+    'Identify flaky tests from a CTRF report file',
+    (yargs) => {
+      return yargs
+        .positional('file', {
+          describe: 'CTRF report file',
+          type: 'string',
+          demandOption: true,
         });
-
-        if (ctrfReportFiles.length === 0) {
-          console.log('No CTRF reports found in the specified directory.');
-          return;
-        }
-
-        if (!fs.existsSync(outputDir)) {
-          fs.mkdirSync(outputDir, { recursive: true });
-          console.log(`Created output directory: ${outputDir}`);
-        }
-
-        const mergedReport = ctrfReportFiles
-          .map((file) => {
-            console.log("Merging report:", file);
-            const filePath = path.join(directoryPath, file);
-            const fileContent = fs.readFileSync(filePath, 'utf8');
-            return JSON.parse(fileContent);
-          })
-          .reduce((acc, curr) => {
-            if (!acc.results) {
-              return curr;
-            }
-
-            acc.results.summary.tests += curr.results.summary.tests;
-            acc.results.summary.passed += curr.results.summary.passed;
-            acc.results.summary.failed += curr.results.summary.failed;
-            acc.results.summary.skipped += curr.results.summary.skipped;
-            acc.results.summary.pending += curr.results.summary.pending;
-            acc.results.summary.other += curr.results.summary.other;
-
-            acc.results.tests.push(...curr.results.tests);
-
-            acc.results.summary.start = Math.min(acc.results.summary.start, curr.results.summary.start);
-            acc.results.summary.stop = Math.max(acc.results.summary.stop, curr.results.summary.stop);
-
-            return acc;
-          }, { results: null });
-
-        fs.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
-
-        if (!keepReports) {
-          ctrfReportFiles.forEach((file) => {
-            const filePath = path.join(directoryPath, file);
-            if (file !== outputFileName) {
-              fs.unlinkSync(filePath);
-            }
-          });
-        }
-
-        console.log('CTRF reports merged successfully.');
-        console.log(`Merged report saved to: ${outputPath}`);
-      } catch (error) {
-        console.error('Error merging CTRF reports:', error);
-      }
+    },
+    async (argv) => {
+      await identifyFlakyTests(argv.file as string);
     }
   )
   .help()

src/flaky.ts

@@ -0,0 +1,29 @@
+import fs from 'fs';
+import path from 'path';
+
+export async function identifyFlakyTests(filePath: string) {
+  try {
+    const resolvedFilePath = path.resolve(filePath);
+
+    if (!fs.existsSync(resolvedFilePath)) {
+      console.error(`The file ${resolvedFilePath} does not exist.`);
+      return;
+    }
+
+    const fileContent = fs.readFileSync(resolvedFilePath, 'utf8');
+    const report = JSON.parse(fileContent);
+
+    const flakyTests = report.results.tests.filter((test: any) => test.flaky === true);
+
+    if (flakyTests.length > 0) {
+      console.log(`Found ${flakyTests.length} flaky test(s):`);
+      flakyTests.forEach((test: any) => {
+        console.log(`- Test Name: ${test.name}, Retries: ${test.retries}`);
+      });
+    } else {
+      console.log(`No flaky tests found in ${resolvedFilePath}.`);
+    }
+  } catch (error) {
+    console.error('Error identifying flaky tests:', error);
+  }
+}

src/merge.ts

@@ -0,0 +1,84 @@
+import fs from 'fs';
+import path from 'path';
+
+export async function mergeReports(directory: string, output: string, outputDir: string, keepReports: boolean) {
+    try {
+        const directoryPath = path.resolve(directory);
+        const outputFileName = output;
+        const resolvedOutputDir = outputDir ? path.resolve(outputDir) : directoryPath;
+        const outputPath = path.join(resolvedOutputDir, outputFileName);
+
+        console.log("Merging CTRF reports...");
+
+        const files = fs.readdirSync(directoryPath);
+
+        files.forEach((file) => {
+            console.log('Found CTRF report file:', file);
+        });
+
+        const ctrfReportFiles = files.filter((file) => {
+            try {
+                const filePath = path.join(directoryPath, file);
+                const fileContent = fs.readFileSync(filePath, 'utf8');
+                const jsonData = JSON.parse(fileContent);
+                return jsonData.hasOwnProperty('results');
+            } catch (error) {
+                console.error(`Error reading JSON file '${file}':`, error);
+                return false;
+            }
+        });
+
+        if (ctrfReportFiles.length === 0) {
+            console.log('No CTRF reports found in the specified directory.');
+            return;
+        }
+
+        if (!fs.existsSync(resolvedOutputDir)) {
+            fs.mkdirSync(resolvedOutputDir, { recursive: true });
+            console.log(`Created output directory: ${resolvedOutputDir}`);
+        }
+
+        const mergedReport = ctrfReportFiles
+            .map((file) => {
+                console.log("Merging report:", file);
+                const filePath = path.join(directoryPath, file);
+                const fileContent = fs.readFileSync(filePath, 'utf8');
+                return JSON.parse(fileContent);
+            })
+            .reduce((acc, curr) => {
+                if (!acc.results) {
+                    return curr;
+                }
+
+                acc.results.summary.tests += curr.results.summary.tests;
+                acc.results.summary.passed += curr.results.summary.passed;
+                acc.results.summary.failed += curr.results.summary.failed;
+                acc.results.summary.skipped += curr.results.summary.skipped;
+                acc.results.summary.pending += curr.results.summary.pending;
+                acc.results.summary.other += curr.results.summary.other;
+
+                acc.results.tests.push(...curr.results.tests);
+
+                acc.results.summary.start = Math.min(acc.results.summary.start, curr.results.summary.start);
+                acc.results.summary.stop = Math.max(acc.results.summary.stop, curr.results.summary.stop);
+
+                return acc;
+            }, { results: null });
+
+        fs.writeFileSync(outputPath, JSON.stringify(mergedReport, null, 2));
+
+        if (!keepReports) {
+            ctrfReportFiles.forEach((file) => {
+                const filePath = path.join(directoryPath, file);
+                if (file !== outputFileName) {
+                    fs.unlinkSync(filePath);
+                }
+            });
+        }
+
+        console.log('CTRF reports merged successfully.');
+        console.log(`Merged report saved to: ${outputPath}`);
+    } catch (error) {
+        console.error('Error merging CTRF reports:', error);
+    }
+}

test-reports/ctrf-report-one.json

@@ -22,8 +22,8 @@
         "stop": 1708979377,
         "rawStatus": "passed",
         "type": "e2e",
-        "retry": 0,
-        "flake": false,
+        "retries": 5,
+        "flaky": true,
         "suite": "My Login application",
         "filePath": "/Users/matthew/projects/personal/ctrf/wdio-tests/test/specs/test.e2e.ts",
         "browser": "chrome 121.0.6167.184"
@@ -38,8 +38,8 @@
         "trace": "Error: Expect $(`#flash`) to have text containing\n\n\u001b[32m- Expected  - 1\u001b[39m\n\u001b[31m+ Received  + 2\u001b[39m\n\n\u001b[32m- You\u001b[7m logged into a secure area!\u001b[27m\u001b[39m\n\u001b[31m+ You\u001b[7mr username is invalid!\u001b[27m\u001b[39m\n\u001b[31m+ ×\u001b[39m\n    at Context.<anonymous> (/Users/matthew/projects/personal/ctrf/wdio-tests/test/specs/test.e2e.ts:19:45)",
         "rawStatus": "failed",
         "type": "e2e",
-        "retry": 0,
-        "flake": false,
+        "retries": 0,
+        "flaky": false,
         "suite": "My Login application",
         "filePath": "/Users/matthew/projects/personal/ctrf/wdio-tests/test/specs/test.e2e.ts",
         "browser": "chrome 121.0.6167.184"