feat: run examples/ci tests in batches

examples/ci/app/ci/ci.test.ts

@@ -1,17 +1,63 @@
-import { test, describe } from "bun:test"
+import { test, describe } from "bun:test";
 import { TEST_ROUTES } from "./constants";
 import { initiateTest } from "./utils";
+import { TestConfig } from "./types";
 
-describe("workflow integration tests", () => {
-  TEST_ROUTES.forEach(testConfig => {
-    test(
-      testConfig.route,
-      async () => {
-        await initiateTest(testConfig.route, testConfig.waitForSeconds)
-      },
-      {
-        timeout: (testConfig.waitForSeconds + 8) * 1000
-      }
-    )
+const BATCH_SIZE = 5
+const BATCH_COUNT = Math.ceil(TEST_ROUTES.length / BATCH_SIZE)
+
+async function runBatch(batch: TestConfig[], batchIndex: number) {
+  const tasks = batch.map((testConfig, index) => async () => {
+    const batchTestIndex = index + 1;
+    try {
+      await initiateTest(testConfig.route, testConfig.waitForSeconds);
+      console.log(`Success: Test ${testConfig.route} (Batch ${batchIndex}, Test ${batchTestIndex})`);
+    } catch (error_) {
+      const error  = error_ as Error
+      console.error(
+        `Failure: Test ${testConfig.route} (Batch ${batchIndex}, Test ${batchTestIndex}) - Error: ${
+          error.message || error
+        }`
+      );
+      throw error;
+    }
   });
-})
+
+  const results = await Promise.allSettled(tasks.map(task => task()));
+  const failures = results.filter(result => result.status === "rejected");
+
+  console.log(
+    `Batch ${batchIndex}/${BATCH_COUNT}: ${results.length - failures.length}/${results.length} passed`
+  );
+
+  if (failures.length) {
+    throw new Error(`batch ${batchIndex} failed.`)
+  }
+
+  return results;
+}
+
+function declareBatchTest(batch: TestConfig[], batchIndex: number) {
+  const maxWaitTime = Math.max(...batch.map(config => config.waitForSeconds));
+  const timeout = (maxWaitTime + 8) * 1000;
+
+  test(
+    `Batch ${batchIndex} Tests`,
+    async () => {
+      await runBatch(batch, batchIndex);
+    },
+    { timeout }
+  );
+}
+
+describe("workflow integration tests", () => {
+
+  // Sort TEST_ROUTES by 'waitForSeconds' (shortest to longest)
+  const sortedRoutes = [...TEST_ROUTES].sort((a, b) => a.waitForSeconds - b.waitForSeconds);
+
+  for (let i = 0; i < sortedRoutes.length; i += BATCH_SIZE) {
+    const batch = sortedRoutes.slice(i, i + BATCH_SIZE);
+    // @ts-expect-error picked TestConfig mismatch
+    declareBatchTest(batch, Math.floor(i / BATCH_SIZE) + 1);
+  }
+});

examples/ci/app/ci/constants.ts

@@ -27,7 +27,7 @@ export const TEST_ROUTES: Pick<TestConfig, "route" | "waitForSeconds">[] = [
   {
     // runs sleep parallel with other steps
     route: "sleepWithoutAwait",
-    waitForSeconds: 18
+    waitForSeconds: 20
   },
   {
     // checks auth

examples/ci/app/test-routes/sleepWithoutAwait/route.ts

@@ -12,20 +12,20 @@ type Invoice = {
 type Charge = {
   invoice: Invoice;
   success: boolean;
+  counter: number
 };
 
 const header = `test-header-foo`
 const headerValue = `header-bar`
 const payload: Invoice = { date: 123, email: "[email protected]", amount: 10 }
 
-let counter = 0;
-const attemptCharge = () => {
+const attemptCharge = (counter: number) => {
   counter += 1;
   if (counter === 3) {
     counter = 0;
-    return true;
+    return { success: true, counter };
   }
-  return false;
+  return { success: false, counter };
 };
 
 export const { POST, GET } = testServe(
@@ -36,13 +36,19 @@ export const { POST, GET } = testServe(
       expect(typeof invoice, typeof payload);
       expect(JSON.stringify(invoice), JSON.stringify(payload));
 
+      let charge: Charge = {
+        success: false,
+        counter: 0,
+        invoice
+      }
+
       for (let index = 0; index < 3; index++) {
-        const charge = await context.run("attemptCharge", () => {
-          const success = attemptCharge();
-          const charge: Charge = { invoice, success };
-          return charge;
+        charge = await context.run("attemptCharge", () => {
+          const { success, counter } = attemptCharge(charge.counter);
+          const newCharge: Charge = { invoice, success, counter };
+          return newCharge;
         });
-
+        
         if (charge.success) {
           const [updateDb, receipt, sleepResult] = await Promise.all([
             context.run("updateDb", () => {
@@ -57,6 +63,7 @@ export const { POST, GET } = testServe(
           expect(updateDb, 10);
           expect(receipt, "[email protected]");
           expect(sleepResult, undefined);
+          console.log("saving");
 
           await saveResult(
             context,
@@ -82,4 +89,4 @@ export const { POST, GET } = testServe(
       [ header ]: headerValue
     }
   }
-) 
+)