perf(cli): run requests parallelly

swordev · Jan 30, 2024 · 823e774 · 823e774
1 parent 7898c13
commit 823e774
Show file tree

Hide file tree

Showing 2 changed files with 26 additions and 22 deletions.
diff --git a/packages/cli/src/repositories/DatatruckRepository.ts b/packages/cli/src/repositories/DatatruckRepository.ts
@@ -1,3 +1,4 @@
+import { runParallel } from "../utils/async";
 import { logExec } from "../utils/cli";
 import { calcFileHash } from "../utils/crypto";
 import { createFs } from "../utils/datatruck/client";
@@ -137,28 +138,31 @@ export class DatatruckRepository extends RepositoryAbstract<DatatruckRepositoryC
     const snapshotNames = await fs.readdir(".");
     const filterPkg = createPkgFilter(data.options.packageNames);
     const filterTask = createTaskFilter(data.options.packageTaskNames);
+    const filterId = (shortId: string) =>
+      !data.options.ids ||
+      data.options.ids.some((id) => shortId.startsWith(id.slice(0, 8)));
+    const preSnapshots: { name: string; metaData?: string }[] = snapshotNames
+      .filter((snapshotName) => {
+        const data = DatatruckRepository.parseSnapshotName(snapshotName);
+        return (
+          data && filterPkg(data.packageName) && filterId(data.snapshotShortId)
+        );
+      })
+      .map((name) => ({ name }));
+
+    await runParallel({
+      items: preSnapshots,
+      concurrency: 5,
+      async onItem({ item }) {
+        item.metaData = await fs.readFileIfExists(`${item.name}/meta.json`);
+      },
+    });
 
-    for (const snapshotName of snapshotNames) {
-      const snapshotNameData =
-        DatatruckRepository.parseSnapshotName(snapshotName);
-      if (!snapshotNameData) continue;
-      if (!filterPkg(snapshotNameData.packageName)) continue;
-
-      if (
-        data.options.ids &&
-        !data.options.ids.some((id) =>
-          snapshotNameData.snapshotShortId.startsWith(id.slice(0, 8)),
-        )
-      )
-        continue;
-
-      const metaData = await fs.readFileIfExists(`${snapshotName}/meta.json`);
+    for (const { name, metaData } of preSnapshots) {
       const meta =
         !!metaData && (await DatatruckRepository.parseMetaData(metaData));
 
-      if (!meta) continue;
-
-      if (!filterTask(meta.task)) continue;
+      if (!meta || !filterTask(meta.task)) continue;
 
       if (
         data.options.ids &&
@@ -172,7 +176,7 @@ export class DatatruckRepository extends RepositoryAbstract<DatatruckRepositoryC
         continue;
 
       snapshots.push({
-        originalId: snapshotName,
+        originalId: name,
         id: meta.id,
         date: meta.date,
         packageName: meta.package,

diff --git a/packages/cli/src/utils/async.ts b/packages/cli/src/utils/async.ts
@@ -5,7 +5,7 @@ type ItemBuffer<T> = Map<T, AbortController>;
 export async function runParallel<T>(options: {
   items: T[];
   concurrency: number;
-  onChange: (data: {
+  onChange?: (data: {
     buffer: ItemBuffer<T>;
     processed: number;
     proccesing: number;
@@ -26,7 +26,7 @@ export async function runParallel<T>(options: {
       const controller = new AbortController();
       buffer.set(item, controller);
       try {
-        await options.onChange({
+        await options.onChange?.({
           processed,
           proccesing: buffer.size,
           buffer,
@@ -48,7 +48,7 @@ export async function runParallel<T>(options: {
         buffer.delete(item);
         processed++;
         await options.onFinished?.();
-        await options.onChange({
+        await options.onChange?.({
           processed,
           proccesing: buffer.size,
           buffer,