From 7a7c144b30a4661c8da151628172d30c50933c0b Mon Sep 17 00:00:00 2001 From: Rares Matei Date: Fri, 10 Jan 2025 15:51:29 +0000 Subject: [PATCH] coordinate post and main hashing method (#85) --- workflow-steps/cache/hashing-utils.ts | 2 +- workflow-steps/cache/main.ts | 19 ++++++++++++++----- workflow-steps/cache/output/main.js | 17 ++++++++++++----- workflow-steps/cache/output/post.js | 18 +++++++++++------- workflow-steps/cache/post.ts | 17 ++++++++++------- 5 files changed, 48 insertions(+), 25 deletions(-) diff --git a/workflow-steps/cache/hashing-utils.ts b/workflow-steps/cache/hashing-utils.ts index e35d3a9..4b34e0a 100644 --- a/workflow-steps/cache/hashing-utils.ts +++ b/workflow-steps/cache/hashing-utils.ts @@ -63,7 +63,7 @@ export function buildCachePaths( inputPaths: string, warnInvalidPaths: boolean = true, ) { - const directories = Array.from( + const directories: string[] = Array.from( new Set( inputPaths .split('\n') diff --git a/workflow-steps/cache/main.ts b/workflow-steps/cache/main.ts index a44683a..9096360 100644 --- a/workflow-steps/cache/main.ts +++ b/workflow-steps/cache/main.ts @@ -26,9 +26,14 @@ if (!inputKey || !inputPaths) { const paths = buildCachePaths(inputPaths, false); const stringifiedPaths = paths.join(','); -const key = hashKey(`${inputKey}|"${stringifiedPaths}"`); -const currentBranchKeys = [key].map((k) => `${currentBranch}-${k}`); -const baseBranchKeys = baseBranch ? [key].map((k) => `${baseBranch}-${k}`) : []; +const key = `${inputKey} | "${stringifiedPaths}"`; +const hashedKey = hashKey(key); +const currentBranchKeys = [hashedKey].map((k) => `${currentBranch}-${k}`); +const baseBranchKeys = baseBranch + ? [hashedKey].map((k) => `${baseBranch}-${k}`) + : []; + +console.log(`Expanded unhashed key is: ${key}\n`); cacheClient .restore( @@ -38,10 +43,14 @@ cacheClient ) .then((resp: RestoreResponse) => { if (resp.success) { - console.log('Found cache entry on hashed key: ' + resp.key); + console.log('Found cache entry on: ' + resp.key); rememberCacheRestorationForPostStep(); } else { - console.log('Cache miss on hashed key: ' + key); + console.log(`Cache miss on:`); + console.log(`- ${currentBranch}-${hashedKey}`); + if (baseBranch && currentBranch !== baseBranch) { + console.log(`- ${baseBranch}-${hashedKey}`); + } } }); diff --git a/workflow-steps/cache/output/main.js b/workflow-steps/cache/output/main.js index 881cb28..f88049f 100644 --- a/workflow-steps/cache/output/main.js +++ b/workflow-steps/cache/output/main.js @@ -6041,19 +6041,26 @@ if (!inputKey || !inputPaths) { } var paths = buildCachePaths(inputPaths, false); var stringifiedPaths = paths.join(","); -var key = hashKey(`${inputKey}|"${stringifiedPaths}"`); -var currentBranchKeys = [key].map((k) => `${currentBranch}-${k}`); -var baseBranchKeys = baseBranch ? [key].map((k) => `${baseBranch}-${k}`) : []; +var key = `${inputKey} | "${stringifiedPaths}"`; +var hashedKey = hashKey(key); +var currentBranchKeys = [hashedKey].map((k) => `${currentBranch}-${k}`); +var baseBranchKeys = baseBranch ? [hashedKey].map((k) => `${baseBranch}-${k}`) : []; +console.log(`Expanded unhashed key is: ${key} +`); cacheClient.restore( new RestoreRequest({ keys: [...currentBranchKeys, ...baseBranchKeys] }) ).then((resp) => { if (resp.success) { - console.log("Found cache entry on hashed key: " + resp.key); + console.log("Found cache entry on: " + resp.key); rememberCacheRestorationForPostStep(); } else { - console.log("Cache miss on hashed key: " + key); + console.log(`Cache miss on:`); + console.log(`- ${currentBranch}-${hashedKey}`); + if (baseBranch && currentBranch !== baseBranch) { + console.log(`- ${baseBranch}-${hashedKey}`); + } } }); function rememberCacheRestorationForPostStep() { diff --git a/workflow-steps/cache/output/post.js b/workflow-steps/cache/output/post.js index 279848f..4dfb575 100644 --- a/workflow-steps/cache/output/post.js +++ b/workflow-steps/cache/output/post.js @@ -6028,21 +6028,25 @@ if (!!cacheWasHit) { ); const paths = buildCachePaths(inputPaths); const stringifiedPaths = paths.join(","); - const key = hashKey(`${inputKey}|${stringifiedPaths}`); - console.log("Storing the following directories..\n" + paths.join("\n")); - console.log(` -Using key..${key}`); + const key = `${inputKey} | "${stringifiedPaths}"`; + const hashedKey = hashKey(key); + console.log(`Expanded unhashed cache key is: ${key}`); + console.log( + `Hashed key that will be used for storage: ${process.env.NX_BRANCH}-${hashedKey}` + ); + console.log("\nDirectories marked for upload:\n" + paths.join("\n")); cacheClient.storeV2( new StoreRequest({ - key, + key: hashedKey, paths }) ).then((r) => { if (r.success) - console.log(`Successfully stored to cache under key ${key}`); + console.log(` +Successfully uploaded marked directories.`); if (r.skipped) console.log( - "Skipped storing to cache, another instance has already started the upload" + "\nSkipped storing to cache, another instance has already started the upload." ); }); } diff --git a/workflow-steps/cache/post.ts b/workflow-steps/cache/post.ts index 35b5bc9..e3987a3 100644 --- a/workflow-steps/cache/post.ts +++ b/workflow-steps/cache/post.ts @@ -24,24 +24,27 @@ if (!!cacheWasHit) { const paths = buildCachePaths(inputPaths); const stringifiedPaths = paths.join(','); - const key = hashKey(`${inputKey}|${stringifiedPaths}`); + const key = `${inputKey} | "${stringifiedPaths}"`; + const hashedKey = hashKey(key); - console.log('Storing the following directories..\n' + paths.join('\n')); - console.log(`\nUsing key..${key}`); + console.log(`Expanded unhashed cache key is: ${key}`); + console.log( + `Hashed key that will be used for storage: ${process.env.NX_BRANCH}-${hashedKey}`, + ); + console.log('\nDirectories marked for upload:\n' + paths.join('\n')); cacheClient .storeV2( new StoreRequest({ - key, + key: hashedKey, paths, }), ) .then((r: StoreResponse) => { - if (r.success) - console.log(`Successfully stored to cache under key ${key}`); + if (r.success) console.log(`\nSuccessfully uploaded marked directories.`); if (r.skipped) console.log( - 'Skipped storing to cache, another instance has already started the upload', + '\nSkipped storing to cache, another instance has already started the upload.', ); }); }