From 07dcce75e4b48e8d131513616469f4e4ee4ed0ea Mon Sep 17 00:00:00 2001 From: metalwarrior665 Date: Tue, 4 Oct 2022 17:01:38 +0200 Subject: [PATCH 1/9] feat: initial add of apify-extra package with dataset functions (build not working yet) --- jest.config.js | 1 + packages/apify-extra/.npmignore | 6 + packages/apify-extra/README.md | 2 + packages/apify-extra/package.json | 64 +++++ packages/apify-extra/src/const.ts | 2 + packages/apify-extra/src/dataset.ts | 339 +++++++++++++++++++++++ packages/apify-extra/src/index.ts | 1 + packages/apify-extra/tsconfig.build.json | 7 + packages/apify-extra/tsconfig.json | 4 + tsconfig.json | 1 + 10 files changed, 427 insertions(+) create mode 100644 packages/apify-extra/.npmignore create mode 100644 packages/apify-extra/README.md create mode 100644 packages/apify-extra/package.json create mode 100644 packages/apify-extra/src/const.ts create mode 100644 packages/apify-extra/src/dataset.ts create mode 100644 packages/apify-extra/src/index.ts create mode 100644 packages/apify-extra/tsconfig.build.json create mode 100644 packages/apify-extra/tsconfig.json diff --git a/jest.config.js b/jest.config.js index e4bdb83789..108491669e 100644 --- a/jest.config.js +++ b/jest.config.js @@ -13,6 +13,7 @@ module.exports = { ], moduleNameMapper: { '^apify$': '/packages/apify/src', + '^apify-extra$': '/packages/apify-extra/src', '^@apify/scraper-tools$': '/packages/scraper-tools/src', }, modulePathIgnorePatterns: [ diff --git a/packages/apify-extra/.npmignore b/packages/apify-extra/.npmignore new file mode 100644 index 0000000000..ada367d72a --- /dev/null +++ b/packages/apify-extra/.npmignore @@ -0,0 +1,6 @@ +node_modules +src +test +coverage +apify_storage +tsconfig.* diff --git a/packages/apify-extra/README.md b/packages/apify-extra/README.md new file mode 100644 index 0000000000..2a4afaba19 --- /dev/null +++ b/packages/apify-extra/README.md @@ -0,0 +1,2 @@ +# Apify Extra + diff --git a/packages/apify-extra/package.json b/packages/apify-extra/package.json new file mode 100644 index 0000000000..d5123f2f3f --- /dev/null +++ b/packages/apify-extra/package.json @@ -0,0 +1,64 @@ +{ + "name": "apify-extra", + "version": "0.0.1", + "description": "Advanced and experimental functionality for Apify Actors and Crawlee. Smaller test coverage. Use with caution!", + "engines": { + "node": ">=16.0.0" + }, + "main": "./dist/index.js", + "module": "./dist/index.mjs", + "types": "./dist/index.d.ts", + "exports": { + ".": { + "import": "./dist/index.mjs", + "require": "./dist/index.js", + "types": "./dist/index.d.ts" + }, + "./package.json": "./package.json" + }, + "keywords": [ + "apify", + "headless", + "chrome", + "puppeteer", + "crawler", + "scraper" + ], + "author": { + "name": "Apify", + "email": "support@apify.com", + "url": "https://apify.com" + }, + "contributors": [ + "Jan Curn ", + "Marek Trunkat ", + "Ondra Urban " + ], + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "git+https://github.com/apify/apify-sdk-js" + }, + "bugs": { + "url": "https://github.com/apify/apify-sdk-js/issues" + }, + "homepage": "https://sdk.apify.com", + "scripts": { + "build": "npm run clean && npm run compile && npm run copy && npm run fixApifyExport", + "clean": "rimraf ./dist", + "compile": "tsc -p tsconfig.build.json && gen-esm-wrapper ./dist/index.js ./dist/index.mjs", + "copy": "ts-node -T ../../scripts/copy.ts --readme=local", + "fixApifyExport": "ts-node -T ../../scripts/temp_fix_apify_exports.ts" + }, + "publishConfig": { + "access": "public" + }, + "dependencies": { + "@types/bluebird": "^3.5.37", + "bluebird": "^3.7.2" + }, + "peerDependencies": { + "apify": ">= 3.0.0", + "crawlee": ">= 3.0.0" + } +} diff --git a/packages/apify-extra/src/const.ts b/packages/apify-extra/src/const.ts new file mode 100644 index 0000000000..4bfc232667 --- /dev/null +++ b/packages/apify-extra/src/const.ts @@ -0,0 +1,2 @@ +export const APIFY_EXTRA_KV_RECORD_PREFIX = 'APIFY-EXTRA-'; +export const APIFY_EXTRA_LOG_PREFIX = '[apify-extra]: '; diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts new file mode 100644 index 0000000000..a3f344e41b --- /dev/null +++ b/packages/apify-extra/src/dataset.ts @@ -0,0 +1,339 @@ +import { Actor } from 'apify'; +import { Dictionary, log, sleep } from 'crawlee'; +import bluebird from 'bluebird'; +import type { Dataset } from 'apify-client'; + +import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const.js'; + +export type DatasetItem = Dictionary; + +export interface ParallelPersistedPushDataOptions { + uploadBatchSize?: number; + uploadSleepMs?: number; + outputDatasetIdOrName?: string; + parallelPushes?: number; +} +/** + * Useful for pushing a large number of items at once + * where migration could introduce duplicates and consume extra CUs + * Only first param is mandatory + */ +export const parallelPersistedPushData = async (items: DatasetItem[], options: ParallelPersistedPushDataOptions = {}) => { + const { + uploadBatchSize = 5000, + uploadSleepMs = 500, + outputDatasetIdOrName = '', + parallelPushes = 1, + } = options; + let isMigrating = false; + Actor.on('migrating', () => { isMigrating = true; }); + Actor.on('aborting', () => { isMigrating = true; }); + + const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT-${outputDatasetIdOrName}`; + let pushedItemsCount: number = (await Actor.getValue(kvRecordName)) || 0; + const dataset = await Actor.openDataset(outputDatasetIdOrName); + + for (let i = pushedItemsCount; i < items.length; i += uploadBatchSize) { + if (isMigrating) { + log.info(`${APIFY_EXTRA_LOG_PREFIX}[parallelPersistedPushData]: Stopping push because of migration`); + // Do nothing + await new Promise(() => {}); + } + const itemsToPush = items.slice(i, i + uploadBatchSize); + + const pushPromises: Promise[] = []; + const parallelizedBatchSize = Math.ceil(itemsToPush.length / parallelPushes); + for (let j = 0; j < parallelPushes; j++) { + const start = j * parallelizedBatchSize; + const end = (j + 1) * parallelizedBatchSize; + const parallelPushChunk = itemsToPush.slice(start, end); + pushPromises.push(dataset.pushData(parallelPushChunk)); + } + // We must update it before awaiting the promises because the push can take time + // and migration can cut us off but the items will already be on the way to dataset + pushedItemsCount += itemsToPush.length; + await Actor.setValue(kvRecordName, pushedItemsCount); + await Promise.all(pushPromises); + await sleep(uploadSleepMs); + } +}; + +interface CalculateLocalOffsetLimitParams { + offset: number; + limit: number; + localStart: number; + batchSize: number; +} + +type CalculateLocalOffsetLimitResult = + null | + { offset: number; limit: number }; + +// Returns either null if offset/limit does not fit the current chunk +// or { offset, limit } object +const calculateLocalOffsetLimit = ({ offset, limit, localStart, batchSize }: CalculateLocalOffsetLimitParams): CalculateLocalOffsetLimitResult => { + const localEnd = localStart + batchSize; + const inputEnd = offset + limit; + + // Offset starts after the current chunk + if (offset >= localEnd) { + return null; + } + // Offset + limit ends before our chunk + if (inputEnd <= localStart) { + return null; + } + + // Now we know that the some data are in the current batch + const calculateLimit = () => { + // limit overflows current batch + if (inputEnd >= localEnd) { + // Now either the offset is less than local start and we do whole batch + if (offset < localStart) { + return batchSize; + } + // Or it is inside the current batch and we slice it from the start (including whole batch) + return localEnd - offset; + // eslint-disable-next-line no-else-return + } else { // Consider (inputEnd < localEnd) Means limit ends inside current batch + if (offset < localStart) { + return inputEnd - localStart; + } + // This means both offset and limit are inside current batch + return inputEnd - offset; + } + }; + + return { + offset: Math.max(localStart, offset), + limit: calculateLimit(), + }; +}; + +const PROCESS_FN_LOADING_STATE_KV_RECORD_KEY = `${APIFY_EXTRA_KV_RECORD_PREFIX}PROCESS-FN-LOADING-STATE`; + +export interface LoadDatasetItemsInParallelOptions { + parallelLoads?: number; + batchSize?: number; + offset?: number; + limit?: number; + concatItems?: boolean; + concatDatasets?: boolean; + debugLog?: boolean; + persistLoadingStateForProcesFn?: boolean; + fields?: string[]; + // If run outside of Apify platform, will fetch local datasets instead. On Apify platform, this is ignored. + loadFromLocalDataset?: boolean; + processFn?: (items: DatasetItem[], params: { datasetId: string; datasetOffset: number }) => Promise; +} + +export type LoadDatasetItemsInParallelResult = + undefined | + DatasetItem[] | + DatasetItem[][]; + +/** +* Loads items from one or many datasets in parallel by chunking the items from each dataset into batches, +* retaining order of both items and datasets. Useful for large loads. +* By default returns one array of items in order of datasets provided. +* By changing concatItems or concatDatasets options, you can get array of arrays (of arrays) back +* Requires bluebird dependency and copy calculateLocalOffsetLimit function!!! +* +* @param {string[]} datasetIds IDs or names of datasets you want to load +* @param {object} options Options with default values. +* If both concatItems and concatDatasets are false, output of this function is an array of datasets containing arrays +* of batches containig array of items. +* concatItems concats all batches of one dataset into one array of items. +* concatDatasets concat all datasets into one array of batches +* Using both concatItems and concatDatasets gives you back a sinlge array of all items in order. +* Both are true by default. +* Data are not returned by fed to the supplied async function on the fly (reduces memory usage) +* Will not load batches that were already processed before migration, does nothing if processFn is not used. +* It does not persist the state inside processFn, that is a responsibillity of the caller (if needed) +* You must not manipulate input parameters (and underlying datasets) between migrations or this will break +*/ +export const loadDatasetItemsInParallel = async ( + datasetIds: string[], + options: LoadDatasetItemsInParallelOptions = {}, +): Promise => { + const { + processFn, + parallelLoads = 20, + batchSize = 50000, + offset = 0, + limit = 999999999, + concatItems = true, + concatDatasets = true, + debugLog = false, + persistLoadingStateForProcesFn = false, + fields, + // Figure out better name since this is useful for datasets by name on platform + loadFromLocalDataset = false, + } = options; + + const LOG_PREFIX = `${APIFY_EXTRA_LOG_PREFIX}[loadDatasetItemsInParallel]:`; + + if (!Actor.isAtHome() && loadFromLocalDataset && fields) { + log.warning(`${LOG_PREFIX} fields option does not work on local datasets`); + } + + const client = Actor.newClient(); + + const loadStart = Date.now(); + + // If we use processFnLoadingState, we skip requests that are done + const createRequestArray = async (processFnLoadingState: any) => { + // We increment for each dataset so we remember their order + let datasetIndex = 0; + + // This array will be used to create promises to run in parallel + const requestInfoArr = []; + + for (const datasetId of datasetIds) { + if (processFnLoadingState && !processFnLoadingState[datasetId]) { + processFnLoadingState[datasetId] = {}; + } + // We get the number of items first and then we precreate request info objects + let datasetInfo: Dataset | undefined; + if (loadFromLocalDataset) { + const dataset = await Actor.openDataset(datasetId); + datasetInfo = await dataset.getInfo() as Dataset; + } else { + datasetInfo = await client.dataset(datasetId).get(); + } + if (!datasetInfo) { + throw new Error(`${LOG_PREFIX} Dataset ${datasetId} was not found`); + } + const { itemCount } = datasetInfo; + if (debugLog) { + log.info(`Dataset ${datasetId} has ${itemCount} items`); + } + const numberOfBatches = Math.ceil(itemCount / batchSize); + + for (let i = 0; i < numberOfBatches; i++) { + const localOffsetLimit = calculateLocalOffsetLimit({ offset, limit, localStart: i * batchSize, batchSize }); + if (!localOffsetLimit) { + continue; + } + + if (processFnLoadingState) { + if (!processFnLoadingState[datasetId][localOffsetLimit.offset]) { + processFnLoadingState[datasetId][localOffsetLimit.offset] = { done: false }; + } else if (processFnLoadingState[datasetId][localOffsetLimit.offset].done) { + if (debugLog) { + log.info(`Batch for dataset ${datasetId}, offset: ${localOffsetLimit.offset} was already processed, skipping...`); + } + continue; + } + } + + requestInfoArr.push({ + index: i, + offset: localOffsetLimit.offset, + limit: localOffsetLimit.limit, + datasetId, + datasetIndex, + }); + } + + datasetIndex++; + } + return requestInfoArr; + }; + + // This is array of arrays. Top level array is for each dataset and inside one entry for each batch (in order) + const loadedBatchedArr: DatasetItem[][][] = []; + + let totalLoaded = 0; + const totalLoadedPerDataset: Record = {}; + + const processFnLoadingState: any = persistLoadingStateForProcesFn + ? (await Actor.getValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY) || {}) + : null; + + Actor.on('persistState', async () => { + await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); + }); + + const requestInfoArr = await createRequestArray(processFnLoadingState); + if (debugLog) { + log.info(`Number of requests to do: ${requestInfoArr.length}`); + } + + // Now we execute all the requests in parallel (with defined concurrency) + await bluebird.map(requestInfoArr, async (requestInfoObj) => { + const { index, datasetId, datasetIndex } = requestInfoObj; + + const getDataOptions = { + offset: requestInfoObj.offset, + limit: requestInfoObj.limit, + fields, + }; + let datasetResult; + if (loadFromLocalDataset) { + // This open should be cached + const dataset = await Actor.openDataset(datasetId); + + if (!Actor.isAtHome()) { + delete getDataOptions.fields; + } + datasetResult = await dataset.getData(getDataOptions); + } else { + datasetResult = await client.dataset(datasetId).listItems(getDataOptions); + } + + const { items } = datasetResult; + + if (!totalLoadedPerDataset[datasetId]) { + totalLoadedPerDataset[datasetId] = 0; + } + + totalLoadedPerDataset[datasetId] += items.length; + totalLoaded += items.length; + + if (debugLog) { + log.info( + `Items loaded from dataset ${datasetId}: ${items.length}, offset: ${requestInfoObj.offset}, + total loaded from dataset ${datasetId}: ${totalLoadedPerDataset[datasetId]}, + total loaded: ${totalLoaded}`, + ); + } + // We either collect the data or we process them on the fly + if (processFn) { + await processFn(items, { datasetId, datasetOffset: requestInfoObj.offset }); + if (processFnLoadingState) { + processFnLoadingState[datasetId][requestInfoObj.offset].done = true; + } + } else { + if (!loadedBatchedArr[datasetIndex]) { + loadedBatchedArr[datasetIndex] = []; + } + // Now we correctly assign the items into the main array + loadedBatchedArr[datasetIndex][index] = items; + } + }, { concurrency: parallelLoads }); + + if (debugLog) { + log.info(`Loading took ${Math.round((Date.now() - loadStart) / 1000)} seconds`); + } + + if (processFnLoadingState) { + await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); + } + + if (processFn) { + return undefined; + } + + let resultItems: DatasetItem[] | DatasetItem[][] | DatasetItem[][][] = loadedBatchedArr; + if (concatItems) { + for (let i = 0; i < loadedBatchedArr.length; i++) { + resultItems[i] = loadedBatchedArr[i].flatMap((item) => item); + } + } + + if (concatDatasets) { + resultItems = loadedBatchedArr.flatMap((item) => item); + } + return loadedBatchedArr; +}; diff --git a/packages/apify-extra/src/index.ts b/packages/apify-extra/src/index.ts new file mode 100644 index 0000000000..d85fc56fe1 --- /dev/null +++ b/packages/apify-extra/src/index.ts @@ -0,0 +1 @@ +export * from './dataset'; diff --git a/packages/apify-extra/tsconfig.build.json b/packages/apify-extra/tsconfig.build.json new file mode 100644 index 0000000000..856db0f210 --- /dev/null +++ b/packages/apify-extra/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.build.json", + "compilerOptions": { + "outDir": "./dist" + }, + "include": ["src/**/*"] +} diff --git a/packages/apify-extra/tsconfig.json b/packages/apify-extra/tsconfig.json new file mode 100644 index 0000000000..52d43eaaa9 --- /dev/null +++ b/packages/apify-extra/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../../tsconfig.json", + "include": ["src/**/*"] +} diff --git a/tsconfig.json b/tsconfig.json index 340a505f87..c208175625 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -4,6 +4,7 @@ "baseUrl": ".", "paths": { "apify": ["packages/apify/src"], + "apify-extra": ["packages/apify-extra/src"], "@apify/scraper-tools": ["packages/scraper-tools/src"] } } From dbac9eeec2dab33462c22c8d09440e632c6c28a5 Mon Sep 17 00:00:00 2001 From: metalwarrior665 Date: Tue, 4 Oct 2022 17:33:55 +0200 Subject: [PATCH 2/9] update package-lock --- package-lock.json | 381 ++++++++++++---------------------------------- 1 file changed, 98 insertions(+), 283 deletions(-) diff --git a/package-lock.json b/package-lock.json index b6c894de6b..a707fab7e9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1340,7 +1340,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true, "engines": { "node": ">=12" }, @@ -1375,7 +1374,6 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, "funding": [ { "type": "github", @@ -1411,7 +1409,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", - "dev": true, "dependencies": { "restore-cursor": "^4.0.0" }, @@ -1426,7 +1423,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.0.0.tgz", "integrity": "sha512-ZksGS2xpa/bYkNzN3BAw1wEjsLV/ZKOf/CCrJ/QOBsxx6fOARIkwTutxp1XIOIohi6HKmOFjMoK/XaqDVUpEEw==", - "dev": true, "engines": { "node": ">= 12" } @@ -1435,7 +1431,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "dev": true, "engines": { "node": ">=12" }, @@ -1447,7 +1442,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", "integrity": "sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==", - "dev": true, "dependencies": { "escape-string-regexp": "^5.0.0", "is-unicode-supported": "^1.2.0" @@ -1489,7 +1483,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", - "dev": true, "engines": { "node": ">=12" }, @@ -1501,7 +1494,6 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", - "dev": true, "engines": { "node": ">=12" }, @@ -1513,7 +1505,6 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz", "integrity": "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==", - "dev": true, "dependencies": { "chalk": "^5.0.0", "is-unicode-supported": "^1.1.0" @@ -1529,7 +1520,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/ora/-/ora-6.1.2.tgz", "integrity": "sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==", - "dev": true, "dependencies": { "bl": "^5.0.0", "chalk": "^5.0.0", @@ -1552,7 +1542,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", - "dev": true, "dependencies": { "onetime": "^5.1.0", "signal-exit": "^3.0.2" @@ -1585,7 +1574,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", - "dev": true, "dependencies": { "ansi-regex": "^6.0.1" }, @@ -1600,7 +1588,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.0.1.tgz", "integrity": "sha512-QFF+ufAqhoYHvoHdajT/Po7KoXVBPXS2bgjIam5isfWJPfIOnQZ50JtUiVvCv/sjgacf3yRrt2ZKUZ/V4itN4g==", - "dev": true, "dependencies": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -4106,7 +4093,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "dev": true, "engines": { "node": ">= 10" } @@ -4620,8 +4606,7 @@ "node_modules/abab": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", - "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", - "dev": true + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" }, "node_modules/abbrev": { "version": "1.1.1", @@ -4763,7 +4748,6 @@ "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, "dependencies": { "type-fest": "^0.21.3" }, @@ -4778,7 +4762,6 @@ "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true, "engines": { "node": ">=10" }, @@ -4790,7 +4773,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "engines": { "node": ">=8" } @@ -4799,7 +4781,6 @@ "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "dependencies": { "color-convert": "^2.0.1" }, @@ -5043,8 +5024,7 @@ "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "node_modules/at-least-node": { "version": "1.0.0", @@ -5247,6 +5227,11 @@ "readable-stream": "^3.4.0" } }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "node_modules/boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -5542,7 +5527,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "dependencies": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -5566,8 +5550,7 @@ "node_modules/chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", - "dev": true + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, "node_modules/cheerio": { "version": "1.0.0-rc.12", @@ -5678,7 +5661,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "dev": true, "dependencies": { "restore-cursor": "^3.1.0" }, @@ -5690,7 +5672,6 @@ "version": "2.6.1", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.6.1.tgz", "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==", - "dev": true, "engines": { "node": ">=6" }, @@ -5762,7 +5743,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", - "dev": true, "engines": { "node": ">= 10" } @@ -5771,7 +5751,6 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -5782,7 +5761,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "dev": true, "engines": { "node": ">=0.8" } @@ -5864,7 +5842,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "dependencies": { "color-name": "~1.1.4" }, @@ -5875,8 +5852,7 @@ "node_modules/color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "node_modules/color-support": { "version": "1.1.3", @@ -5910,7 +5886,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "dependencies": { "delayed-stream": "~1.0.0" }, @@ -6317,14 +6292,12 @@ "node_modules/cssom": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", - "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", - "dev": true + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "node_modules/cssstyle": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", - "dev": true, "dependencies": { "cssom": "~0.3.6" }, @@ -6362,7 +6335,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==", - "dev": true, "dependencies": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -6469,8 +6441,7 @@ "node_modules/deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" }, "node_modules/deepmerge": { "version": "4.2.2", @@ -6540,7 +6511,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -6684,7 +6654,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", - "dev": true, "dependencies": { "webidl-conversions": "^7.0.0" }, @@ -6786,8 +6755,7 @@ "node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, "node_modules/encoding": { "version": "0.1.13", @@ -6952,7 +6920,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", - "dev": true, "dependencies": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -6974,7 +6941,6 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "dev": true, "dependencies": { "prelude-ls": "~1.1.2", "type-check": "~0.3.2" @@ -6987,7 +6953,6 @@ "version": "0.8.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dev": true, "dependencies": { "deep-is": "~0.1.3", "fast-levenshtein": "~2.0.6", @@ -7004,7 +6969,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "dev": true, "engines": { "node": ">= 0.8.0" } @@ -7013,7 +6977,6 @@ "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", - "dev": true, "dependencies": { "prelude-ls": "~1.1.2" }, @@ -7561,7 +7524,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true, "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -7598,7 +7560,6 @@ "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true, "engines": { "node": ">=4.0" } @@ -7607,7 +7568,6 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -7683,7 +7643,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "dev": true, "dependencies": { "chardet": "^0.7.0", "iconv-lite": "^0.4.24", @@ -7697,7 +7656,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3" }, @@ -7780,8 +7738,7 @@ "node_modules/fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, "node_modules/fastq": { "version": "1.13.0", @@ -7813,7 +7770,6 @@ "version": "1.5.2", "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.5.2.tgz", "integrity": "sha512-WOn21V8AhyE1QqVfPIVxe3tupJacq1xGkPTB4iagT6o+P2cAgEOOwIxMftr4+ZCTI6d551ij9j61DFr0nsP2uQ==", - "dev": true, "engines": { "node": ">= 0.4.0" } @@ -7822,7 +7778,6 @@ "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "dev": true, "dependencies": { "escape-string-regexp": "^1.0.5" }, @@ -7837,7 +7792,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, "engines": { "node": ">=0.8.0" } @@ -7979,7 +7933,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -8133,7 +8086,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true, "engines": { "node": "6.* || 8.* || >= 10.*" } @@ -8591,7 +8543,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -8603,7 +8554,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -8621,7 +8571,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true, "engines": { "node": ">=8" } @@ -8711,7 +8660,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", - "dev": true, "dependencies": { "whatwg-encoding": "^2.0.0" }, @@ -8765,7 +8713,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dev": true, "dependencies": { "@tootallnate/once": "2", "agent-base": "6", @@ -8921,7 +8868,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", - "dev": true, "dependencies": { "pkg-dir": "^4.2.0", "resolve-cwd": "^3.0.0" @@ -9234,7 +9180,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", - "dev": true, "engines": { "node": ">=8" } @@ -9319,8 +9264,7 @@ "node_modules/is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", - "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", - "dev": true + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" }, "node_modules/is-regex": { "version": "1.1.4", @@ -9423,7 +9367,6 @@ "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "dev": true, "engines": { "node": ">=10" }, @@ -10854,8 +10797,7 @@ "node_modules/lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "node_modules/lodash.isequal": { "version": "4.5.0", @@ -10888,7 +10830,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -11287,7 +11228,6 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true, "engines": { "node": ">=6" } @@ -11550,8 +11490,7 @@ "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "dev": true + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, "node_modules/nanoid": { "version": "3.3.4", @@ -12021,8 +11960,7 @@ "node_modules/nwsapi": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.2.tgz", - "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==", - "dev": true + "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==" }, "node_modules/nx": { "version": "15.0.13", @@ -12331,7 +12269,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, "dependencies": { "mimic-fn": "^2.1.0" }, @@ -12380,7 +12317,6 @@ "version": "5.4.1", "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", - "dev": true, "dependencies": { "bl": "^4.1.0", "chalk": "^4.1.0", @@ -12403,7 +12339,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -12660,7 +12595,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/parent-require/-/parent-require-1.0.0.tgz", "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==", - "dev": true, "engines": { "node": ">= 0.4.0" } @@ -13616,7 +13550,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -13660,7 +13593,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, "dependencies": { "resolve-from": "^5.0.0" }, @@ -13672,7 +13604,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, "engines": { "node": ">=8" } @@ -13713,7 +13644,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "dev": true, "dependencies": { "onetime": "^5.1.0", "signal-exit": "^3.0.2" @@ -13785,7 +13715,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "dev": true, "engines": { "node": ">=0.12.0" } @@ -13864,7 +13793,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", - "dev": true, "dependencies": { "xmlchars": "^2.2.0" }, @@ -14120,7 +14048,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true, + "devOptional": true, "engines": { "node": ">=0.10.0" } @@ -14281,7 +14209,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -14294,14 +14221,12 @@ "node_modules/string-width/node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "node_modules/string-width/node_modules/is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true, "engines": { "node": ">=8" } @@ -14357,7 +14282,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "dependencies": { "ansi-regex": "^5.0.1" }, @@ -14428,7 +14352,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "dependencies": { "has-flag": "^4.0.0" }, @@ -14451,8 +14374,7 @@ "node_modules/symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", - "dev": true + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, "node_modules/tar": { "version": "6.1.12", @@ -14601,7 +14523,6 @@ "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, "dependencies": { "os-tmpdir": "~1.0.2" }, @@ -14662,7 +14583,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", - "dev": true, "dependencies": { "punycode": "^2.1.1" }, @@ -15220,7 +15140,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", - "dev": true, "dependencies": { "xml-name-validator": "^4.0.0" }, @@ -15247,7 +15166,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "dev": true, "dependencies": { "defaults": "^1.0.3" } @@ -15256,7 +15174,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "dev": true, "engines": { "node": ">=12" } @@ -15265,7 +15182,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", - "dev": true, "dependencies": { "iconv-lite": "0.6.3" }, @@ -15277,7 +15193,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", - "dev": true, "engines": { "node": ">=12" } @@ -15286,7 +15201,6 @@ "version": "11.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", - "dev": true, "dependencies": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -15339,7 +15253,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -15354,7 +15267,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "dependencies": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -15553,7 +15465,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", - "dev": true, "engines": { "node": ">=12" } @@ -15561,8 +15472,7 @@ "node_modules/xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", - "dev": true + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, "node_modules/xtend": { "version": "4.0.2", @@ -15577,7 +15487,6 @@ "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true, "engines": { "node": ">=10" } @@ -15599,7 +15508,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/yargonaut/-/yargonaut-1.1.4.tgz", "integrity": "sha512-rHgFmbgXAAzl+1nngqOcwEljqHGG9uUZoPjsdZEs1w5JW9RXYzrSvH/u70C1JE5qFi0qjsdhnUX/dJRpWqitSA==", - "dev": true, "dependencies": { "chalk": "^1.1.1", "figlet": "^1.1.1", @@ -15610,7 +15518,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -15619,7 +15526,6 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true, "engines": { "node": ">=0.10.0" } @@ -15628,7 +15534,6 @@ "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", - "dev": true, "dependencies": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -15644,7 +15549,6 @@ "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, "engines": { "node": ">=0.8.0" } @@ -15653,7 +15557,6 @@ "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "dev": true, "dependencies": { "ansi-regex": "^2.0.0" }, @@ -15665,7 +15568,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true, "engines": { "node": ">=0.8.0" } @@ -15715,7 +15617,6 @@ "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true, "engines": { "node": ">=12" } @@ -15845,6 +15746,21 @@ "node": ">=16.0.0" } }, + "packages/apify-extra": { + "version": "0.0.1", + "license": "Apache-2.0", + "dependencies": { + "@types/bluebird": "^3.5.37", + "bluebird": "^3.7.2" + }, + "engines": { + "node": ">=16.0.0" + }, + "peerDependencies": { + "apify": ">= 3.0.0", + "crawlee": ">= 3.0.0" + } + }, "packages/scraper-tools": { "name": "@apify/scraper-tools", "version": "1.1.1", @@ -16881,8 +16797,7 @@ "ansi-regex": { "version": "6.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", - "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", - "dev": true + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==" }, "ansi-styles": { "version": "6.2.1", @@ -16905,7 +16820,6 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, "requires": { "base64-js": "^1.3.1", "ieee754": "^1.2.1" @@ -16921,7 +16835,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", - "dev": true, "requires": { "restore-cursor": "^4.0.0" } @@ -16929,20 +16842,17 @@ "cli-width": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-4.0.0.tgz", - "integrity": "sha512-ZksGS2xpa/bYkNzN3BAw1wEjsLV/ZKOf/CCrJ/QOBsxx6fOARIkwTutxp1XIOIohi6HKmOFjMoK/XaqDVUpEEw==", - "dev": true + "integrity": "sha512-ZksGS2xpa/bYkNzN3BAw1wEjsLV/ZKOf/CCrJ/QOBsxx6fOARIkwTutxp1XIOIohi6HKmOFjMoK/XaqDVUpEEw==" }, "escape-string-regexp": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-5.0.0.tgz", - "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==", - "dev": true + "integrity": "sha512-/veY75JbMK4j1yjvuUxuVsiS/hr/4iHs9FTT6cgTexxdE0Ly/glccBAkloH/DofkjRbZU3bnoj38mOmhkZ0lHw==" }, "figures": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/figures/-/figures-5.0.0.tgz", "integrity": "sha512-ej8ksPF4x6e5wvK9yevct0UCXh8TTFlWGVLlgjZuoBH1HwjIfKE/IdL5mq89sFA7zELi1VhKpmtDnrs7zWyeyg==", - "dev": true, "requires": { "escape-string-regexp": "^5.0.0", "is-unicode-supported": "^1.2.0" @@ -16974,20 +16884,17 @@ "is-interactive": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", - "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", - "dev": true + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==" }, "is-unicode-supported": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-1.3.0.tgz", - "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==", - "dev": true + "integrity": "sha512-43r2mRvz+8JRIKnWJ+3j8JtjRKZ6GmjzfaE/qiBJnikNnYv/6bagRJ1kUhNk8R5EX/GkobD+r+sfxCPJsiKBLQ==" }, "log-symbols": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz", "integrity": "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==", - "dev": true, "requires": { "chalk": "^5.0.0", "is-unicode-supported": "^1.1.0" @@ -16997,7 +16904,6 @@ "version": "6.1.2", "resolved": "https://registry.npmjs.org/ora/-/ora-6.1.2.tgz", "integrity": "sha512-EJQ3NiP5Xo94wJXIzAyOtSb0QEIAUu7m8t6UZ9krbz0vAJqr92JpcK/lEXg91q6B9pEGqrykkd2EQplnifDSBw==", - "dev": true, "requires": { "bl": "^5.0.0", "chalk": "^5.0.0", @@ -17014,7 +16920,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", - "dev": true, "requires": { "onetime": "^5.1.0", "signal-exit": "^3.0.2" @@ -17024,7 +16929,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, "requires": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -17035,7 +16939,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.0.1.tgz", "integrity": "sha512-cXNxvT8dFNRVfhVME3JAe98mkXDYN2O1l7jmcwMnOslDeESg1rF/OZMtK0nRAhiari1unG5cD4jG3rapUAkLbw==", - "dev": true, "requires": { "ansi-regex": "^6.0.1" } @@ -17044,7 +16947,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.0.1.tgz", "integrity": "sha512-QFF+ufAqhoYHvoHdajT/Po7KoXVBPXS2bgjIam5isfWJPfIOnQZ50JtUiVvCv/sjgacf3yRrt2ZKUZ/V4itN4g==", - "dev": true, "requires": { "ansi-styles": "^6.1.0", "string-width": "^5.0.1", @@ -19094,8 +18996,7 @@ "@tootallnate/once": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/@tootallnate/once/-/once-2.0.0.tgz", - "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==", - "dev": true + "integrity": "sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A==" }, "@tsconfig/node10": { "version": "1.0.9", @@ -19506,8 +19407,7 @@ "abab": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/abab/-/abab-2.0.6.tgz", - "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==", - "dev": true + "integrity": "sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==" }, "abbrev": { "version": "1.1.1", @@ -19659,7 +19559,6 @@ "version": "4.3.2", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-4.3.2.tgz", "integrity": "sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ==", - "dev": true, "requires": { "type-fest": "^0.21.3" }, @@ -19667,22 +19566,19 @@ "type-fest": { "version": "0.21.3", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.21.3.tgz", - "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==", - "dev": true + "integrity": "sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w==" } } }, "ansi-regex": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", - "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true + "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==" }, "ansi-styles": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dev": true, "requires": { "color-convert": "^2.0.1" } @@ -19895,8 +19791,7 @@ "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", - "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", - "dev": true + "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==" }, "at-least-node": { "version": "1.0.0", @@ -20051,6 +19946,11 @@ "readable-stream": "^3.4.0" } }, + "bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, "boolbase": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", @@ -20261,7 +20161,6 @@ "version": "4.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.2.tgz", "integrity": "sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==", - "dev": true, "requires": { "ansi-styles": "^4.1.0", "supports-color": "^7.1.0" @@ -20276,8 +20175,7 @@ "chardet": { "version": "0.7.0", "resolved": "https://registry.npmjs.org/chardet/-/chardet-0.7.0.tgz", - "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==", - "dev": true + "integrity": "sha512-mT8iDcrh03qDGRRmoA2hmBJnxpllMR+0/0qlzjqZES6NdiWDcZkCNAk4rPFZ9Q85r27unkiNNg8ZOiwZXBHwcA==" }, "cheerio": { "version": "1.0.0-rc.12", @@ -20361,7 +20259,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", - "dev": true, "requires": { "restore-cursor": "^3.1.0" } @@ -20369,8 +20266,7 @@ "cli-spinners": { "version": "2.6.1", "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.6.1.tgz", - "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==", - "dev": true + "integrity": "sha512-x/5fWmGMnbKQAaNwN+UZlV79qBLM9JFnJuJ03gIi5whrob0xV0ofNVHy9DhwGdsMJQc2OKv0oGmLzvaqvAVv+g==" }, "cli-truncate": { "version": "3.1.0", @@ -20413,14 +20309,12 @@ "cli-width": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-3.0.0.tgz", - "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==", - "dev": true + "integrity": "sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw==" }, "cliui": { "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", - "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -20430,8 +20324,7 @@ "clone": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", - "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==", - "dev": true + "integrity": "sha512-JQHZ2QMW6l3aH/j6xCqQThY/9OH4D/9ls34cgkUBiEeocRTU04tHfKPBsUK1PqZCUQM7GiA0IIXJSuXHI64Kbg==" }, "clone-deep": { "version": "4.0.1", @@ -20495,7 +20388,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dev": true, "requires": { "color-name": "~1.1.4" } @@ -20503,8 +20395,7 @@ "color-name": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", - "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", - "dev": true + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, "color-support": { "version": "1.1.3", @@ -20532,7 +20423,6 @@ "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", - "dev": true, "requires": { "delayed-stream": "~1.0.0" } @@ -20837,14 +20727,12 @@ "cssom": { "version": "0.5.0", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.5.0.tgz", - "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==", - "dev": true + "integrity": "sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw==" }, "cssstyle": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/cssstyle/-/cssstyle-2.3.0.tgz", "integrity": "sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A==", - "dev": true, "requires": { "cssom": "~0.3.6" }, @@ -20852,8 +20740,7 @@ "cssom": { "version": "0.3.8", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", - "dev": true + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" } } }, @@ -20878,7 +20765,6 @@ "version": "3.0.2", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-3.0.2.tgz", "integrity": "sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ==", - "dev": true, "requires": { "abab": "^2.0.6", "whatwg-mimetype": "^3.0.0", @@ -20952,8 +20838,7 @@ "deep-is": { "version": "0.1.4", "resolved": "https://registry.npmjs.org/deep-is/-/deep-is-0.1.4.tgz", - "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==", - "dev": true + "integrity": "sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==" }, "deepmerge": { "version": "4.2.2", @@ -20998,8 +20883,7 @@ "delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", - "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", - "dev": true + "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, "delegates": { "version": "1.0.0", @@ -21110,7 +20994,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/domexception/-/domexception-4.0.0.tgz", "integrity": "sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw==", - "dev": true, "requires": { "webidl-conversions": "^7.0.0" } @@ -21182,8 +21065,7 @@ "emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", - "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", - "dev": true + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==" }, "encoding": { "version": "0.1.13", @@ -21309,7 +21191,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/escodegen/-/escodegen-2.0.0.tgz", "integrity": "sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw==", - "dev": true, "requires": { "esprima": "^4.0.1", "estraverse": "^5.2.0", @@ -21322,7 +21203,6 @@ "version": "0.3.0", "resolved": "https://registry.npmjs.org/levn/-/levn-0.3.0.tgz", "integrity": "sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA==", - "dev": true, "requires": { "prelude-ls": "~1.1.2", "type-check": "~0.3.2" @@ -21332,7 +21212,6 @@ "version": "0.8.3", "resolved": "https://registry.npmjs.org/optionator/-/optionator-0.8.3.tgz", "integrity": "sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA==", - "dev": true, "requires": { "deep-is": "~0.1.3", "fast-levenshtein": "~2.0.6", @@ -21345,14 +21224,12 @@ "prelude-ls": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/prelude-ls/-/prelude-ls-1.1.2.tgz", - "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==", - "dev": true + "integrity": "sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w==" }, "type-check": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.3.2.tgz", "integrity": "sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg==", - "dev": true, "requires": { "prelude-ls": "~1.1.2" } @@ -21785,8 +21662,7 @@ "esprima": { "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "dev": true + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" }, "esquery": { "version": "1.4.0", @@ -21809,14 +21685,12 @@ "estraverse": { "version": "5.3.0", "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.3.0.tgz", - "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==", - "dev": true + "integrity": "sha512-MMdARuVEQziNTeJD8DgMqmhwR11BRQ/cBP+pLtYdSTnf3MIO8fFeiINEbX36ZdNlfU/7A9f3gUw49B3oQsvwBA==" }, "esutils": { "version": "2.0.3", "resolved": "https://registry.npmjs.org/esutils/-/esutils-2.0.3.tgz", - "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==", - "dev": true + "integrity": "sha512-kVscqXk4OCp68SZ0dkgEKVi6/8ij300KBWTJq32P/dYeWTSwK41WyTxalN1eRmA5Z9UU/LX9D7FWSmV9SAYx6g==" }, "event-stream": { "version": "3.3.4", @@ -21877,7 +21751,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", "integrity": "sha512-hMQ4CX1p1izmuLYyZqLMO/qGNw10wSv9QDCPfzXfyFrOaCSSoRfqE1Kf1s5an66J5JZC62NewG+mK49jOCtQew==", - "dev": true, "requires": { "chardet": "^0.7.0", "iconv-lite": "^0.4.24", @@ -21888,7 +21761,6 @@ "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", - "dev": true, "requires": { "safer-buffer": ">= 2.1.2 < 3" } @@ -21954,8 +21826,7 @@ "fast-levenshtein": { "version": "2.0.6", "resolved": "https://registry.npmjs.org/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz", - "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", - "dev": true + "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==" }, "fastq": { "version": "1.13.0", @@ -21986,14 +21857,12 @@ "figlet": { "version": "1.5.2", "resolved": "https://registry.npmjs.org/figlet/-/figlet-1.5.2.tgz", - "integrity": "sha512-WOn21V8AhyE1QqVfPIVxe3tupJacq1xGkPTB4iagT6o+P2cAgEOOwIxMftr4+ZCTI6d551ij9j61DFr0nsP2uQ==", - "dev": true + "integrity": "sha512-WOn21V8AhyE1QqVfPIVxe3tupJacq1xGkPTB4iagT6o+P2cAgEOOwIxMftr4+ZCTI6d551ij9j61DFr0nsP2uQ==" }, "figures": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/figures/-/figures-3.2.0.tgz", "integrity": "sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg==", - "dev": true, "requires": { "escape-string-regexp": "^1.0.5" }, @@ -22001,8 +21870,7 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" } } }, @@ -22093,7 +21961,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", "integrity": "sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww==", - "dev": true, "requires": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", @@ -22212,8 +22079,7 @@ "get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", - "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==", - "dev": true + "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==" }, "get-intrinsic": { "version": "1.1.3", @@ -22562,7 +22428,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/has-ansi/-/has-ansi-2.0.0.tgz", "integrity": "sha512-C8vBJ8DwUCx19vhm7urhTuUsr4/IyP6l4VzNQDv+ryHQObW3TTTp9yB68WpYgRe2bbaGuZ/se74IqFeVnMnLZg==", - "dev": true, "requires": { "ansi-regex": "^2.0.0" }, @@ -22570,8 +22435,7 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" } } }, @@ -22584,8 +22448,7 @@ "has-flag": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-4.0.0.tgz", - "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==", - "dev": true + "integrity": "sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ==" }, "has-property-descriptors": { "version": "1.0.0", @@ -22650,7 +22513,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz", "integrity": "sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA==", - "dev": true, "requires": { "whatwg-encoding": "^2.0.0" } @@ -22691,7 +22553,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz", "integrity": "sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w==", - "dev": true, "requires": { "@tootallnate/once": "2", "agent-base": "6", @@ -22793,7 +22654,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/import-local/-/import-local-3.1.0.tgz", "integrity": "sha512-ASB07uLtnDs1o6EHjKpX34BKYDSqnFerfTOJL2HvMqF70LnxpjkzDB8J44oT9pu4AMPkQwf8jl6szgvNd2tRIg==", - "dev": true, "requires": { "pkg-dir": "^4.2.0", "resolve-cwd": "^3.0.0" @@ -23025,8 +22885,7 @@ "is-interactive": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", - "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", - "dev": true + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==" }, "is-lambda": { "version": "1.0.1", @@ -23081,8 +22940,7 @@ "is-potential-custom-element-name": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz", - "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==", - "dev": true + "integrity": "sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ==" }, "is-regex": { "version": "1.1.4", @@ -23154,8 +23012,7 @@ "is-unicode-supported": { "version": "0.1.0", "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", - "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", - "dev": true + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==" }, "is-valid-identifier": { "version": "2.0.2", @@ -24240,8 +24097,7 @@ "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", - "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==", - "dev": true + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, "lodash.isequal": { "version": "4.5.0", @@ -24274,7 +24130,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", - "dev": true, "requires": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" @@ -24574,8 +24429,7 @@ "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", - "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", - "dev": true + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==" }, "mimic-response": { "version": "3.1.0", @@ -24772,8 +24626,7 @@ "mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", - "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==", - "dev": true + "integrity": "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA==" }, "nanoid": { "version": "3.3.4", @@ -25140,8 +24993,7 @@ "nwsapi": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/nwsapi/-/nwsapi-2.2.2.tgz", - "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==", - "dev": true + "integrity": "sha512-90yv+6538zuvUMnN+zCr8LuV6bPFdq50304114vJYJ8RDyK8D5O9Phpbd6SZWgI7PwzmmfN1upeOJlvybDSgCw==" }, "nx": { "version": "15.0.13", @@ -25370,7 +25222,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", - "dev": true, "requires": { "mimic-fn": "^2.1.0" } @@ -25404,7 +25255,6 @@ "version": "5.4.1", "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.1.tgz", "integrity": "sha512-5b6Y85tPxZZ7QytO+BQzysW31HJku27cRIlkbAXaNx+BdcVi+LlRFmVXzeF6a7JCwJpyw5c4b+YSVImQIrBpuQ==", - "dev": true, "requires": { "bl": "^4.1.0", "chalk": "^4.1.0", @@ -25420,8 +25270,7 @@ "os-tmpdir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-tmpdir/-/os-tmpdir-1.0.2.tgz", - "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==", - "dev": true + "integrity": "sha512-D2FR03Vir7FIu45XBY20mTb+/ZSWB00sjU9jdQXt83gDrI4Ztz5Fs7/yy74g2N5SVQY4xY1qDr4rNddwYRVX0g==" }, "ow": { "version": "0.28.2", @@ -25597,8 +25446,7 @@ "parent-require": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/parent-require/-/parent-require-1.0.0.tgz", - "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==", - "dev": true + "integrity": "sha512-2MXDNZC4aXdkkap+rBBMv0lUsfJqvX5/2FiYYnfCnorZt3Pk06/IOR5KeaoghgS2w07MLWgjbsnyaq6PdHn2LQ==" }, "parse-conflict-json": { "version": "2.0.2", @@ -26315,8 +26163,7 @@ "require-directory": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz", - "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==", - "dev": true + "integrity": "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q==" }, "require-from-string": { "version": "2.0.2", @@ -26348,7 +26195,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/resolve-cwd/-/resolve-cwd-3.0.0.tgz", "integrity": "sha512-OrZaX2Mb+rJCpH/6CpSqt9xFVpN++x01XnN2ie9g6P5/3xelLAkXWVADpdz1IHD/KFfEXyE6V0U01OQ3UO2rEg==", - "dev": true, "requires": { "resolve-from": "^5.0.0" } @@ -26356,8 +26202,7 @@ "resolve-from": { "version": "5.0.0", "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true + "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==" }, "resolve-global": { "version": "1.0.0", @@ -26386,7 +26231,6 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", - "dev": true, "requires": { "onetime": "^5.1.0", "signal-exit": "^3.0.2" @@ -26440,8 +26284,7 @@ "run-async": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/run-async/-/run-async-2.4.1.tgz", - "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==", - "dev": true + "integrity": "sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ==" }, "run-parallel": { "version": "1.2.0", @@ -26486,7 +26329,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/saxes/-/saxes-6.0.0.tgz", "integrity": "sha512-xAg7SOnEhrm5zI3puOOKyy1OMcMlIJZYNJY7xLBwSze0UjhPLnWfj2GF2EpT0jmzaJKIWKHLsaSSajf35bcYnA==", - "dev": true, "requires": { "xmlchars": "^2.2.0" } @@ -26668,7 +26510,7 @@ "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==", - "dev": true + "devOptional": true }, "source-map-support": { "version": "0.5.13", @@ -26810,7 +26652,6 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -26820,14 +26661,12 @@ "emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", - "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "dev": true + "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==" }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", - "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "dev": true + "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==" } } }, @@ -26873,7 +26712,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "requires": { "ansi-regex": "^5.0.1" } @@ -26920,7 +26758,6 @@ "version": "7.2.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-7.2.0.tgz", "integrity": "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==", - "dev": true, "requires": { "has-flag": "^4.0.0" } @@ -26934,8 +26771,7 @@ "symbol-tree": { "version": "3.2.4", "resolved": "https://registry.npmjs.org/symbol-tree/-/symbol-tree-3.2.4.tgz", - "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==", - "dev": true + "integrity": "sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw==" }, "tar": { "version": "6.1.12", @@ -27067,7 +26903,6 @@ "version": "0.0.33", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.0.33.tgz", "integrity": "sha512-jRCJlojKnZ3addtTOjdIqoRuPEKBvNXcGYqzO6zWZX8KfKEpnGY5jfggJQ3EjKuu8D4bJRr0y+cYJFmYbImXGw==", - "dev": true, "requires": { "os-tmpdir": "~1.0.2" } @@ -27115,7 +26950,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/tr46/-/tr46-3.0.0.tgz", "integrity": "sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA==", - "dev": true, "requires": { "punycode": "^2.1.1" } @@ -27501,7 +27335,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz", "integrity": "sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg==", - "dev": true, "requires": { "xml-name-validator": "^4.0.0" } @@ -27525,7 +27358,6 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", "integrity": "sha512-XHPEwS0q6TaxcvG85+8EYkbiCux2XtWG2mkc47Ng2A77BQu9+DqIOJldST4HgPkuea7dvKSj5VgX3P1d4rW8Tg==", - "dev": true, "requires": { "defaults": "^1.0.3" } @@ -27533,14 +27365,12 @@ "webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", - "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==", - "dev": true + "integrity": "sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g==" }, "whatwg-encoding": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz", "integrity": "sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg==", - "dev": true, "requires": { "iconv-lite": "0.6.3" } @@ -27548,14 +27378,12 @@ "whatwg-mimetype": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz", - "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==", - "dev": true + "integrity": "sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q==" }, "whatwg-url": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-11.0.0.tgz", "integrity": "sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ==", - "dev": true, "requires": { "tr46": "^3.0.0", "webidl-conversions": "^7.0.0" @@ -27595,8 +27423,7 @@ "word-wrap": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.3.tgz", - "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==", - "dev": true + "integrity": "sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ==" }, "wordwrap": { "version": "1.0.0", @@ -27608,7 +27435,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", - "dev": true, "requires": { "ansi-styles": "^4.0.0", "string-width": "^4.1.0", @@ -27754,14 +27580,12 @@ "xml-name-validator": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/xml-name-validator/-/xml-name-validator-4.0.0.tgz", - "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==", - "dev": true + "integrity": "sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw==" }, "xmlchars": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/xmlchars/-/xmlchars-2.2.0.tgz", - "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", - "dev": true + "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==" }, "xtend": { "version": "4.0.2", @@ -27772,8 +27596,7 @@ "y18n": { "version": "5.0.8", "resolved": "https://registry.npmjs.org/y18n/-/y18n-5.0.8.tgz", - "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==", - "dev": true + "integrity": "sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==" }, "yallist": { "version": "4.0.0", @@ -27789,7 +27612,6 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/yargonaut/-/yargonaut-1.1.4.tgz", "integrity": "sha512-rHgFmbgXAAzl+1nngqOcwEljqHGG9uUZoPjsdZEs1w5JW9RXYzrSvH/u70C1JE5qFi0qjsdhnUX/dJRpWqitSA==", - "dev": true, "requires": { "chalk": "^1.1.1", "figlet": "^1.1.1", @@ -27799,20 +27621,17 @@ "ansi-regex": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", - "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==", - "dev": true + "integrity": "sha512-TIGnTpdo+E3+pCyAluZvtED5p5wCqLdezCyhPZzKPcxvFplEt4i+W7OONCKgeZFT3+y5NZZfOOS/Bdcanm1MYA==" }, "ansi-styles": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", - "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==", - "dev": true + "integrity": "sha512-kmCevFghRiWM7HB5zTPULl4r9bVFSWjz62MhqizDGUrq2NWuNMQyuv4tHHoKJHs69M/MF64lEcHdYIocrdWQYA==" }, "chalk": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", "integrity": "sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==", - "dev": true, "requires": { "ansi-styles": "^2.2.1", "escape-string-regexp": "^1.0.2", @@ -27824,14 +27643,12 @@ "escape-string-regexp": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true + "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==" }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", "integrity": "sha512-VhumSSbBqDTP8p2ZLKj40UjBCV4+v8bUSEpUb4KjRgWk9pbqGF4REFj6KEagidb2f/M6AzC0EmFyDNGaw9OCzg==", - "dev": true, "requires": { "ansi-regex": "^2.0.0" } @@ -27839,8 +27656,7 @@ "supports-color": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", - "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==", - "dev": true + "integrity": "sha512-KKNVtd6pCYgPIKU4cp2733HWYCpplQhddZLBUryaAHou723x+FRzQ5Df824Fj+IyyuiQTRoub4SnIFfIcrp70g==" } } }, @@ -27873,8 +27689,7 @@ "yargs-parser": { "version": "21.1.1", "resolved": "https://registry.npmjs.org/yargs-parser/-/yargs-parser-21.1.1.tgz", - "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==", - "dev": true + "integrity": "sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==" } } }, From 12a5847db8e18e026371c89b0c17166085954c80 Mon Sep 17 00:00:00 2001 From: metalwarrior665 Date: Tue, 4 Oct 2022 20:18:04 +0200 Subject: [PATCH 3/9] fix: improve functionality and docs of parallelPersistedPushData --- packages/apify-extra/src/dataset.ts | 53 +++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts index a3f344e41b..6b26141011 100644 --- a/packages/apify-extra/src/dataset.ts +++ b/packages/apify-extra/src/dataset.ts @@ -1,5 +1,5 @@ import { Actor } from 'apify'; -import { Dictionary, log, sleep } from 'crawlee'; +import { Dictionary, log } from 'crawlee'; import bluebird from 'bluebird'; import type { Dataset } from 'apify-client'; @@ -8,38 +8,65 @@ import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const.js export type DatasetItem = Dictionary; export interface ParallelPersistedPushDataOptions { + /** + * Number of items to be pushed in one push call. + * Should not be higher than 1000 to ensure each push call finishes when migration happens. + * + * @default 1000 + */ uploadBatchSize?: number; - uploadSleepMs?: number; + /** + * Optional dataset ID or name to push into. If not provided, default dataset is used. + */ outputDatasetIdOrName?: string; + /** + * Number of push calls to be done in parallel. Apify API should handle up to 30 parallel requests but better be careful. + * + * @default 10 + */ parallelPushes?: number; + /** + * You must provide idempotency key if you want to call this function multiple times in the saem run. + * The key should only contain letters and numbers. + */ + idempotencyKey?: string; } /** - * Useful for pushing a large number of items at once - * where migration could introduce duplicates and consume extra CUs - * Only first param is mandatory + * Pushes items to dataset in parallel while also resuming correctly after Actor migration. + * This should be used when you want to push a lot of items to dataset at once. + * If you call this function multiple times in the same run, + * you must provide different idempotencyKey option for each call. */ export const parallelPersistedPushData = async (items: DatasetItem[], options: ParallelPersistedPushDataOptions = {}) => { const { - uploadBatchSize = 5000, - uploadSleepMs = 500, + uploadBatchSize = 1000, outputDatasetIdOrName = '', - parallelPushes = 1, + parallelPushes = 10, + idempotencyKey = '', } = options; let isMigrating = false; Actor.on('migrating', () => { isMigrating = true; }); Actor.on('aborting', () => { isMigrating = true; }); - const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT-${outputDatasetIdOrName}`; - let pushedItemsCount: number = (await Actor.getValue(kvRecordName)) || 0; + const sanitizedIdempotencyKey = idempotencyKey.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 30); + + const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT${outputDatasetIdOrName}${sanitizedIdempotencyKey}`; + let pushedItemsCount: number = await Actor.getValue(kvRecordName) || 0; const dataset = await Actor.openDataset(outputDatasetIdOrName); - for (let i = pushedItemsCount; i < items.length; i += uploadBatchSize) { + Actor.on('persistState', async () => { + await Actor.setValue(kvRecordName, pushedItemsCount); + }); + + const fullBatchSize = uploadBatchSize * parallelPushes; + + for (let i = pushedItemsCount; i < items.length; i += fullBatchSize) { if (isMigrating) { log.info(`${APIFY_EXTRA_LOG_PREFIX}[parallelPersistedPushData]: Stopping push because of migration`); // Do nothing await new Promise(() => {}); } - const itemsToPush = items.slice(i, i + uploadBatchSize); + const itemsToPush = items.slice(i, i + fullBatchSize); const pushPromises: Promise[] = []; const parallelizedBatchSize = Math.ceil(itemsToPush.length / parallelPushes); @@ -52,9 +79,7 @@ export const parallelPersistedPushData = async (items: DatasetItem[], options: P // We must update it before awaiting the promises because the push can take time // and migration can cut us off but the items will already be on the way to dataset pushedItemsCount += itemsToPush.length; - await Actor.setValue(kvRecordName, pushedItemsCount); await Promise.all(pushPromises); - await sleep(uploadSleepMs); } }; From bf1843a49e2854b101db7c8cc2d4078249b409a0 Mon Sep 17 00:00:00 2001 From: metalwarrior665 Date: Tue, 4 Oct 2022 20:32:07 +0200 Subject: [PATCH 4/9] fix: use DatasetInfo type from crawlee, not apify-client --- packages/apify-extra/src/dataset.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts index 6b26141011..a2d2f429e4 100644 --- a/packages/apify-extra/src/dataset.ts +++ b/packages/apify-extra/src/dataset.ts @@ -1,7 +1,7 @@ import { Actor } from 'apify'; import { Dictionary, log } from 'crawlee'; +import type { DatasetInfo } from '@crawlee/types'; import bluebird from 'bluebird'; -import type { Dataset } from 'apify-client'; import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const.js'; @@ -219,10 +219,10 @@ export const loadDatasetItemsInParallel = async ( processFnLoadingState[datasetId] = {}; } // We get the number of items first and then we precreate request info objects - let datasetInfo: Dataset | undefined; + let datasetInfo: DatasetInfo | undefined; if (loadFromLocalDataset) { const dataset = await Actor.openDataset(datasetId); - datasetInfo = await dataset.getInfo() as Dataset; + datasetInfo = await dataset.getInfo(); } else { datasetInfo = await client.dataset(datasetId).get(); } From b289368ce1ecfda111a32b0995b79ca43e0a7560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Wed, 19 Oct 2022 15:26:11 +0200 Subject: [PATCH 5/9] feat: add dataset-extra (ideas) --- packages/apify-extra/package.json | 1 + packages/apify-extra/src/dataset.ts | 560 +++++++++++++++------------- 2 files changed, 297 insertions(+), 264 deletions(-) diff --git a/packages/apify-extra/package.json b/packages/apify-extra/package.json index d5123f2f3f..bc618093dc 100644 --- a/packages/apify-extra/package.json +++ b/packages/apify-extra/package.json @@ -54,6 +54,7 @@ "access": "public" }, "dependencies": { + "apify": "^3.0.0", "@types/bluebird": "^3.5.37", "bluebird": "^3.7.2" }, diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts index a2d2f429e4..0e456918c8 100644 --- a/packages/apify-extra/src/dataset.ts +++ b/packages/apify-extra/src/dataset.ts @@ -1,11 +1,10 @@ -import { Actor } from 'apify'; -import { Dictionary, log } from 'crawlee'; +import { Actor, Dataset as OriginalDataset } from 'apify'; +import { log } from 'crawlee'; import type { DatasetInfo } from '@crawlee/types'; -import bluebird from 'bluebird'; -import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const.js'; +import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const'; -export type DatasetItem = Dictionary; +export type DatasetItem = Exclude[0], any[]>; export interface ParallelPersistedPushDataOptions { /** @@ -15,10 +14,6 @@ export interface ParallelPersistedPushDataOptions { * @default 1000 */ uploadBatchSize?: number; - /** - * Optional dataset ID or name to push into. If not provided, default dataset is used. - */ - outputDatasetIdOrName?: string; /** * Number of push calls to be done in parallel. Apify API should handle up to 30 parallel requests but better be careful. * @@ -26,62 +21,305 @@ export interface ParallelPersistedPushDataOptions { */ parallelPushes?: number; /** - * You must provide idempotency key if you want to call this function multiple times in the saem run. + * You must provide idempotency key if you want to call this function multiple times in the same run. * The key should only contain letters and numbers. */ idempotencyKey?: string; } -/** - * Pushes items to dataset in parallel while also resuming correctly after Actor migration. - * This should be used when you want to push a lot of items to dataset at once. - * If you call this function multiple times in the same run, - * you must provide different idempotencyKey option for each call. - */ -export const parallelPersistedPushData = async (items: DatasetItem[], options: ParallelPersistedPushDataOptions = {}) => { - const { - uploadBatchSize = 1000, - outputDatasetIdOrName = '', - parallelPushes = 10, - idempotencyKey = '', - } = options; - let isMigrating = false; - Actor.on('migrating', () => { isMigrating = true; }); - Actor.on('aborting', () => { isMigrating = true; }); - - const sanitizedIdempotencyKey = idempotencyKey.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 30); - - const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT${outputDatasetIdOrName}${sanitizedIdempotencyKey}`; - let pushedItemsCount: number = await Actor.getValue(kvRecordName) || 0; - const dataset = await Actor.openDataset(outputDatasetIdOrName); - - Actor.on('persistState', async () => { - await Actor.setValue(kvRecordName, pushedItemsCount); - }); - - const fullBatchSize = uploadBatchSize * parallelPushes; - - for (let i = pushedItemsCount; i < items.length; i += fullBatchSize) { - if (isMigrating) { - log.info(`${APIFY_EXTRA_LOG_PREFIX}[parallelPersistedPushData]: Stopping push because of migration`); - // Do nothing - await new Promise(() => {}); + +async function waitForCompletion(promises: (() => Promise)[], maxConcurrency: number): Promise { + async function worker() { + let job; + /* eslint-disable-next-line no-cond-assign */ + while (job = promises.shift()) await job(); + } + + await Promise.all([...new Array(maxConcurrency)].map(() => worker())); +} + +export class Dataset extends OriginalDataset { + /** + * Returns batches of items to be pushed to dataset. + * @param data Items to be pushed to dataset. + * @param batchSize Number of items to be pushed in one push call. + * @param batchCount Number of batches to be returned. + * @returns Array of batches of items to be pushed to dataset. + */ + private getBatches(data: DatasetItem | DatasetItem[], batchSize: number, batchCount: number) { + if (!Array.isArray(data)) { + return [[data]]; } - const itemsToPush = items.slice(i, i + fullBatchSize); - - const pushPromises: Promise[] = []; - const parallelizedBatchSize = Math.ceil(itemsToPush.length / parallelPushes); - for (let j = 0; j < parallelPushes; j++) { - const start = j * parallelizedBatchSize; - const end = (j + 1) * parallelizedBatchSize; - const parallelPushChunk = itemsToPush.slice(start, end); - pushPromises.push(dataset.pushData(parallelPushChunk)); + return [...Array(batchCount).keys()] + .map((x: number) => data.slice(x * batchSize, (x + 1) * batchSize)) + .filter((x) => x.length !== 0); + } + + /** + * Push data to dataset in parallel while also resuming correctly after Actor migration. + * This should be used when you want to push a lot of items to dataset at once. + * If you call this function multiple times in the same run, + * you must provide different idempotencyKey option for each call. + */ + public async pushDataParallel(data: DatasetItem | DatasetItem[], options: ParallelPersistedPushDataOptions = {}) { + if (!Array.isArray(data)) { + return this.pushData(data); + } + + const { + uploadBatchSize = 1000, + parallelPushes = 10, + idempotencyKey = '', + } = options; + + let isMigrating = false; + Actor.on('migrating', () => { isMigrating = true; }); + Actor.on('aborting', () => { isMigrating = true; }); + + const sanitizedIdempotencyKey = idempotencyKey.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 30); + + const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT${this.name}${sanitizedIdempotencyKey}`; + let pushedItemsCount = await Actor.getValue(kvRecordName) ?? 0; + + Actor.on('persistState', async () => { + await Actor.setValue(kvRecordName, pushedItemsCount); + }); + + const stepSize = uploadBatchSize * parallelPushes; + + for (let i = pushedItemsCount; i < data.length; i += stepSize) { + if (isMigrating) { + log.info(`${APIFY_EXTRA_LOG_PREFIX}[pushDataParallel]: Stopping push because of migration`); + // hang indefinitely until migration is done + await new Promise(() => {}); + } + + const itemsToPush = data.slice(i, i + stepSize); + const batches = this.getBatches(itemsToPush, uploadBatchSize, parallelPushes); + const pushPromises = batches.map(this.pushData); + + // We must update it before awaiting the promises because the push can take time + // and migration can cut us off but the items will already be on the way to dataset + pushedItemsCount += itemsToPush.length; + await Promise.all(pushPromises); } - // We must update it before awaiting the promises because the push can take time - // and migration can cut us off but the items will already be on the way to dataset - pushedItemsCount += itemsToPush.length; - await Promise.all(pushPromises); + }; + + public async forEachParallel(func: (item: DatasetItem) => void, options: any = {}) { + const { parallelLoads = 20, batchSize = 50000 } = options; + + const { itemCount } = await this.getInfo() ?? { itemCount: 0 }; + + return waitForCompletion( + [...new Array(Math.ceil(itemCount / batchSize))] + .map((_, i) => () => this.forEach(func, { limit: batchSize, offset: batchSize * i })), + parallelLoads); } -}; + + // /** + // * Loads items from one or many datasets in parallel by chunking the items from each dataset into batches, + // * retaining order of both items and datasets. Useful for large loads. + // * By default returns one array of items in order of datasets provided. + // * By changing concatItems or concatDatasets options, you can get array of arrays (of arrays) back + // * Requires bluebird dependency and copy calculateLocalOffsetLimit function!!! + // * + // * @param {string[]} datasetIds IDs or names of datasets you want to load + // * @param {object} options Options with default values. + // * If both concatItems and concatDatasets are false, output of this function is an array of datasets containing arrays + // * of batches containig array of items. + // * concatItems concats all batches of one dataset into one array of items. + // * concatDatasets concat all datasets into one array of batches + // * Using both concatItems and concatDatasets gives you back a sinlge array of all items in order. + // * Both are true by default. + // * Data are not returned by fed to the supplied async function on the fly (reduces memory usage) + // * Will not load batches that were already processed before migration, does nothing if processFn is not used. + // * It does not persist the state inside processFn, that is a responsibillity of the caller (if needed) + // * You must not manipulate input parameters (and underlying datasets) between migrations or this will break + // */ + // public static async getDataParallel( + // datasetIds: string[], + // options: LoadDatasetItemsInParallelOptions = {}, + // ): Promise { + // const { + // processFn, + // parallelLoads = 20, + // batchSize = 50000, + // offset = 0, + // limit = 999999999, + // concatItems = true, + // concatDatasets = true, + // debugLog = false, + // persistLoadingStateForProcesFn = false, + // fields, + // // Figure out better name since this is useful for datasets by name on platform + // loadFromLocalDataset = false, + // } = options; + + // const LOG_PREFIX = `${APIFY_EXTRA_LOG_PREFIX}[loadDatasetItemsInParallel]:`; + + // if (!Actor.isAtHome() && loadFromLocalDataset && fields) { + // log.warning(`${LOG_PREFIX} fields option does not work on local datasets`); + // } + + // const client = Actor.apifyClient; + // const loadStart = Date.now(); + + // // If we use processFnLoadingState, we skip requests that are done + // const createRequestArray = async (processFnLoadingState: any) => { + // // We increment for each dataset so we remember their order + // let datasetIndex = 0; + + // // This array will be used to create promises to run in parallel + // const requestInfoArr = []; + + // for (const datasetId of datasetIds) { + // if (processFnLoadingState && !processFnLoadingState[datasetId]) { + // processFnLoadingState[datasetId] = {}; + // } + // // We get the number of items first and then we precreate request info objects + // let datasetInfo: DatasetInfo | undefined; + // if (loadFromLocalDataset) { + // const dataset = await Actor.openDataset(datasetId); + // datasetInfo = await dataset.getInfo(); + // } else { + // datasetInfo = await client.dataset(datasetId).get(); + // } + // if (!datasetInfo) { + // throw new Error(`${LOG_PREFIX} Dataset ${datasetId} was not found`); + // } + // const { itemCount } = datasetInfo; + // if (debugLog) { + // log.info(`Dataset ${datasetId} has ${itemCount} items`); + // } + // const numberOfBatches = Math.ceil(itemCount / batchSize); + + // for (let i = 0; i < numberOfBatches; i++) { + // const localOffsetLimit = calculateLocalOffsetLimit({ offset, limit, localStart: i * batchSize, batchSize }); + // if (!localOffsetLimit) { + // continue; + // } + + // if (processFnLoadingState) { + // if (!processFnLoadingState[datasetId][localOffsetLimit.offset]) { + // processFnLoadingState[datasetId][localOffsetLimit.offset] = { done: false }; + // } else if (processFnLoadingState[datasetId][localOffsetLimit.offset].done) { + // if (debugLog) { + // log.info(`Batch for dataset ${datasetId}, offset: ${localOffsetLimit.offset} was already processed, skipping...`); + // } + // continue; + // } + // } + + // requestInfoArr.push({ + // index: i, + // offset: localOffsetLimit.offset, + // limit: localOffsetLimit.limit, + // datasetId, + // datasetIndex, + // }); + // } + + // datasetIndex++; + // } + // return requestInfoArr; + // }; + + // // This is array of arrays. Top level array is for each dataset and inside one entry for each batch (in order) + // const loadedBatchedArr: DatasetItem[][][] = []; + + // let totalLoaded = 0; + // const totalLoadedPerDataset: Record = {}; + + // const processFnLoadingState: any = persistLoadingStateForProcesFn + // ? (await Actor.getValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY) || {}) + // : null; + + // Actor.on('persistState', async () => { + // await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); + // }); + + // const requestInfoArr = await createRequestArray(processFnLoadingState); + // if (debugLog) { + // log.info(`Number of requests to do: ${requestInfoArr.length}`); + // } + + // // Now we execute all the requests in parallel (with defined concurrency) + // await concurrentWait(requestInfoArr.map((requestInfoObj) => async () => { + // const { index, datasetId, datasetIndex } = requestInfoObj; + + // const getDataOptions = { + // offset: requestInfoObj.offset, + // limit: requestInfoObj.limit, + // fields, + // }; + // let datasetResult; + // if (loadFromLocalDataset) { + // // This open should be cached + // const dataset = await Actor.openDataset(datasetId); + + // if (!Actor.isAtHome()) { + // delete getDataOptions.fields; + // } + // datasetResult = await dataset.getData(getDataOptions); + // } else { + // datasetResult = await client.dataset(datasetId).listItems(getDataOptions); + // } + + // const { items } = datasetResult; + + // if (!totalLoadedPerDataset[datasetId]) { + // totalLoadedPerDataset[datasetId] = 0; + // } + + // totalLoadedPerDataset[datasetId] += items.length; + // totalLoaded += items.length; + + // if (debugLog) { + // log.info( + // `Items loaded from dataset ${datasetId}: ${items.length}, offset: ${requestInfoObj.offset}, + // total loaded from dataset ${datasetId}: ${totalLoadedPerDataset[datasetId]}, + // total loaded: ${totalLoaded}`, + // ); + // } + // // We either collect the data or we process them on the fly + // if (processFn) { + // await processFn(items, { datasetId, datasetOffset: requestInfoObj.offset }); + // if (processFnLoadingState) { + // processFnLoadingState[datasetId][requestInfoObj.offset].done = true; + // } + // } else { + // if (!loadedBatchedArr[datasetIndex]) { + // loadedBatchedArr[datasetIndex] = []; + // } + // // Now we correctly assign the items into the main array + // loadedBatchedArr[datasetIndex][index] = items; + // } + // }), parallelLoads); + + // if (debugLog) { + // log.info(`Loading took ${Math.round((Date.now() - loadStart) / 1000)} seconds`); + // } + + // if (processFnLoadingState) { + // await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); + // } + + // if (processFn) { + // return undefined; + // } + + // let resultItems: DatasetItem[] | DatasetItem[][] | DatasetItem[][][] = loadedBatchedArr; + // if (concatItems) { + // for (let i = 0; i < loadedBatchedArr.length; i++) { + // resultItems[i] = loadedBatchedArr[i].flatMap((item) => item); + // } + // } + +// if (concatDatasets) { +// resultItems = loadedBatchedArr.flatMap((item) => item); +// } +// return loadedBatchedArr; +// }; +} interface CalculateLocalOffsetLimitParams { offset: number; @@ -156,209 +394,3 @@ export type LoadDatasetItemsInParallelResult = undefined | DatasetItem[] | DatasetItem[][]; - -/** -* Loads items from one or many datasets in parallel by chunking the items from each dataset into batches, -* retaining order of both items and datasets. Useful for large loads. -* By default returns one array of items in order of datasets provided. -* By changing concatItems or concatDatasets options, you can get array of arrays (of arrays) back -* Requires bluebird dependency and copy calculateLocalOffsetLimit function!!! -* -* @param {string[]} datasetIds IDs or names of datasets you want to load -* @param {object} options Options with default values. -* If both concatItems and concatDatasets are false, output of this function is an array of datasets containing arrays -* of batches containig array of items. -* concatItems concats all batches of one dataset into one array of items. -* concatDatasets concat all datasets into one array of batches -* Using both concatItems and concatDatasets gives you back a sinlge array of all items in order. -* Both are true by default. -* Data are not returned by fed to the supplied async function on the fly (reduces memory usage) -* Will not load batches that were already processed before migration, does nothing if processFn is not used. -* It does not persist the state inside processFn, that is a responsibillity of the caller (if needed) -* You must not manipulate input parameters (and underlying datasets) between migrations or this will break -*/ -export const loadDatasetItemsInParallel = async ( - datasetIds: string[], - options: LoadDatasetItemsInParallelOptions = {}, -): Promise => { - const { - processFn, - parallelLoads = 20, - batchSize = 50000, - offset = 0, - limit = 999999999, - concatItems = true, - concatDatasets = true, - debugLog = false, - persistLoadingStateForProcesFn = false, - fields, - // Figure out better name since this is useful for datasets by name on platform - loadFromLocalDataset = false, - } = options; - - const LOG_PREFIX = `${APIFY_EXTRA_LOG_PREFIX}[loadDatasetItemsInParallel]:`; - - if (!Actor.isAtHome() && loadFromLocalDataset && fields) { - log.warning(`${LOG_PREFIX} fields option does not work on local datasets`); - } - - const client = Actor.newClient(); - - const loadStart = Date.now(); - - // If we use processFnLoadingState, we skip requests that are done - const createRequestArray = async (processFnLoadingState: any) => { - // We increment for each dataset so we remember their order - let datasetIndex = 0; - - // This array will be used to create promises to run in parallel - const requestInfoArr = []; - - for (const datasetId of datasetIds) { - if (processFnLoadingState && !processFnLoadingState[datasetId]) { - processFnLoadingState[datasetId] = {}; - } - // We get the number of items first and then we precreate request info objects - let datasetInfo: DatasetInfo | undefined; - if (loadFromLocalDataset) { - const dataset = await Actor.openDataset(datasetId); - datasetInfo = await dataset.getInfo(); - } else { - datasetInfo = await client.dataset(datasetId).get(); - } - if (!datasetInfo) { - throw new Error(`${LOG_PREFIX} Dataset ${datasetId} was not found`); - } - const { itemCount } = datasetInfo; - if (debugLog) { - log.info(`Dataset ${datasetId} has ${itemCount} items`); - } - const numberOfBatches = Math.ceil(itemCount / batchSize); - - for (let i = 0; i < numberOfBatches; i++) { - const localOffsetLimit = calculateLocalOffsetLimit({ offset, limit, localStart: i * batchSize, batchSize }); - if (!localOffsetLimit) { - continue; - } - - if (processFnLoadingState) { - if (!processFnLoadingState[datasetId][localOffsetLimit.offset]) { - processFnLoadingState[datasetId][localOffsetLimit.offset] = { done: false }; - } else if (processFnLoadingState[datasetId][localOffsetLimit.offset].done) { - if (debugLog) { - log.info(`Batch for dataset ${datasetId}, offset: ${localOffsetLimit.offset} was already processed, skipping...`); - } - continue; - } - } - - requestInfoArr.push({ - index: i, - offset: localOffsetLimit.offset, - limit: localOffsetLimit.limit, - datasetId, - datasetIndex, - }); - } - - datasetIndex++; - } - return requestInfoArr; - }; - - // This is array of arrays. Top level array is for each dataset and inside one entry for each batch (in order) - const loadedBatchedArr: DatasetItem[][][] = []; - - let totalLoaded = 0; - const totalLoadedPerDataset: Record = {}; - - const processFnLoadingState: any = persistLoadingStateForProcesFn - ? (await Actor.getValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY) || {}) - : null; - - Actor.on('persistState', async () => { - await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); - }); - - const requestInfoArr = await createRequestArray(processFnLoadingState); - if (debugLog) { - log.info(`Number of requests to do: ${requestInfoArr.length}`); - } - - // Now we execute all the requests in parallel (with defined concurrency) - await bluebird.map(requestInfoArr, async (requestInfoObj) => { - const { index, datasetId, datasetIndex } = requestInfoObj; - - const getDataOptions = { - offset: requestInfoObj.offset, - limit: requestInfoObj.limit, - fields, - }; - let datasetResult; - if (loadFromLocalDataset) { - // This open should be cached - const dataset = await Actor.openDataset(datasetId); - - if (!Actor.isAtHome()) { - delete getDataOptions.fields; - } - datasetResult = await dataset.getData(getDataOptions); - } else { - datasetResult = await client.dataset(datasetId).listItems(getDataOptions); - } - - const { items } = datasetResult; - - if (!totalLoadedPerDataset[datasetId]) { - totalLoadedPerDataset[datasetId] = 0; - } - - totalLoadedPerDataset[datasetId] += items.length; - totalLoaded += items.length; - - if (debugLog) { - log.info( - `Items loaded from dataset ${datasetId}: ${items.length}, offset: ${requestInfoObj.offset}, - total loaded from dataset ${datasetId}: ${totalLoadedPerDataset[datasetId]}, - total loaded: ${totalLoaded}`, - ); - } - // We either collect the data or we process them on the fly - if (processFn) { - await processFn(items, { datasetId, datasetOffset: requestInfoObj.offset }); - if (processFnLoadingState) { - processFnLoadingState[datasetId][requestInfoObj.offset].done = true; - } - } else { - if (!loadedBatchedArr[datasetIndex]) { - loadedBatchedArr[datasetIndex] = []; - } - // Now we correctly assign the items into the main array - loadedBatchedArr[datasetIndex][index] = items; - } - }, { concurrency: parallelLoads }); - - if (debugLog) { - log.info(`Loading took ${Math.round((Date.now() - loadStart) / 1000)} seconds`); - } - - if (processFnLoadingState) { - await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); - } - - if (processFn) { - return undefined; - } - - let resultItems: DatasetItem[] | DatasetItem[][] | DatasetItem[][][] = loadedBatchedArr; - if (concatItems) { - for (let i = 0; i < loadedBatchedArr.length; i++) { - resultItems[i] = loadedBatchedArr[i].flatMap((item) => item); - } - } - - if (concatDatasets) { - resultItems = loadedBatchedArr.flatMap((item) => item); - } - return loadedBatchedArr; -}; From a935be539c786b0a0f2cabc0f6647daee15abc9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Fri, 28 Oct 2022 11:51:09 +0200 Subject: [PATCH 6/9] feat: forEachParallel, stub tests --- package-lock.json | 73 ++---- packages/apify-extra/src/dataset.ts | 334 +++++----------------------- test/apify/extra-dataset.test.ts | 62 ++++++ 3 files changed, 136 insertions(+), 333 deletions(-) create mode 100644 test/apify/extra-dataset.test.ts diff --git a/package-lock.json b/package-lock.json index a707fab7e9..e2803734f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1325,7 +1325,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-6.0.0.tgz", "integrity": "sha512-IG23inYII3dWlU2EyiAiGj6Bwal5GzsgPMwjYGvc1HPE2dgbj4ZB5ToWBKSquKw74nB3TIuOwaI6/jSULzfgrw==", - "dev": true, "dependencies": { "type-fest": "^3.0.0" }, @@ -1351,7 +1350,6 @@ "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true, "engines": { "node": ">=12" }, @@ -1363,7 +1361,6 @@ "version": "5.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", - "dev": true, "dependencies": { "buffer": "^6.0.3", "inherits": "^2.0.4", @@ -1397,7 +1394,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.1.2.tgz", "integrity": "sha512-E5CkT4jWURs1Vy5qGJye+XwCkNj7Od3Af7CP6SujMetSMkLs8Do2RWJK5yx1wamHV/op8Rz+9rltjaTQWDnEFQ==", - "dev": true, "engines": { "node": "^12.17.0 || ^14.13 || >=16.0.0" }, @@ -1457,7 +1453,6 @@ "version": "9.1.4", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.1.4.tgz", "integrity": "sha512-9hiJxE5gkK/cM2d1mTEnuurGTAoHebbkX0BYl3h7iEg7FYfuNIom+nDfBCSWtvSnoSrWCeBxqqBZu26xdlJlXA==", - "dev": true, "dependencies": { "ansi-escapes": "^6.0.0", "chalk": "^5.1.2", @@ -1557,7 +1552,6 @@ "version": "5.1.2", "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", - "dev": true, "dependencies": { "eastasianwidth": "^0.2.0", "emoji-regex": "^9.2.2", @@ -4630,7 +4624,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz", "integrity": "sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q==", - "dev": true, "dependencies": { "acorn": "^8.1.0", "acorn-walk": "^8.0.2" @@ -4649,7 +4642,6 @@ "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "dev": true, "engines": { "node": ">=0.4.0" } @@ -4836,6 +4828,10 @@ "follow-redirects": "^1.14.0" } }, + "node_modules/apify-extra": { + "resolved": "packages/apify-extra", + "link": true + }, "node_modules/aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -5751,6 +5747,7 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -6308,8 +6305,7 @@ "node_modules/cssstyle/node_modules/cssom": { "version": "0.3.8", "resolved": "https://registry.npmjs.org/cssom/-/cssom-0.3.8.tgz", - "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==", - "dev": true + "integrity": "sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg==" }, "node_modules/csv-stringify": { "version": "6.2.1", @@ -6415,8 +6411,7 @@ "node_modules/decimal.js": { "version": "10.4.2", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.2.tgz", - "integrity": "sha512-ic1yEvwT6GuvaYwBLLY6/aFFgjZdySKTE8en/fkU3QICTmRtgtSlFn0u0BXN06InZwtfCelR7j8LRiDI/02iGA==", - "dev": true + "integrity": "sha512-ic1yEvwT6GuvaYwBLLY6/aFFgjZdySKTE8en/fkU3QICTmRtgtSlFn0u0BXN06InZwtfCelR7j8LRiDI/02iGA==" }, "node_modules/decompress-response": { "version": "6.0.0", @@ -6456,7 +6451,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "dev": true, "dependencies": { "clone": "^1.0.2" }, @@ -6717,8 +6711,7 @@ "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "node_modules/ejs": { "version": "3.1.8", @@ -12498,7 +12491,6 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true, "engines": { "node": ">=6" } @@ -12675,7 +12667,6 @@ "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true, "engines": { "node": ">=8" } @@ -12778,7 +12769,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, "dependencies": { "find-up": "^4.0.0" }, @@ -12790,7 +12780,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, "dependencies": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -12803,7 +12792,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, "dependencies": { "p-locate": "^4.1.0" }, @@ -12815,7 +12803,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, "dependencies": { "p-try": "^2.0.0" }, @@ -12830,7 +12817,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, "dependencies": { "p-limit": "^2.2.0" }, @@ -13746,7 +13732,6 @@ "version": "7.5.7", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", "integrity": "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", - "dev": true, "dependencies": { "tslib": "^2.1.0" } @@ -15603,7 +15588,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "dependencies": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", @@ -16789,7 +16773,6 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-6.0.0.tgz", "integrity": "sha512-IG23inYII3dWlU2EyiAiGj6Bwal5GzsgPMwjYGvc1HPE2dgbj4ZB5ToWBKSquKw74nB3TIuOwaI6/jSULzfgrw==", - "dev": true, "requires": { "type-fest": "^3.0.0" } @@ -16802,14 +16785,12 @@ "ansi-styles": { "version": "6.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", - "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", - "dev": true + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==" }, "bl": { "version": "5.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", - "dev": true, "requires": { "buffer": "^6.0.3", "inherits": "^2.0.4", @@ -16828,8 +16809,7 @@ "chalk": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.1.2.tgz", - "integrity": "sha512-E5CkT4jWURs1Vy5qGJye+XwCkNj7Od3Af7CP6SujMetSMkLs8Do2RWJK5yx1wamHV/op8Rz+9rltjaTQWDnEFQ==", - "dev": true + "integrity": "sha512-E5CkT4jWURs1Vy5qGJye+XwCkNj7Od3Af7CP6SujMetSMkLs8Do2RWJK5yx1wamHV/op8Rz+9rltjaTQWDnEFQ==" }, "cli-cursor": { "version": "4.0.0", @@ -16862,7 +16842,6 @@ "version": "9.1.4", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-9.1.4.tgz", "integrity": "sha512-9hiJxE5gkK/cM2d1mTEnuurGTAoHebbkX0BYl3h7iEg7FYfuNIom+nDfBCSWtvSnoSrWCeBxqqBZu26xdlJlXA==", - "dev": true, "requires": { "ansi-escapes": "^6.0.0", "chalk": "^5.1.2", @@ -19425,7 +19404,6 @@ "version": "7.0.1", "resolved": "https://registry.npmjs.org/acorn-globals/-/acorn-globals-7.0.1.tgz", "integrity": "sha512-umOSDSDrfHbTNPuNpC2NSnnA3LUrqpevPb4T9jRx4MagXNS0rs+gwiTcAvqCRmsD6utzsrzNt+ebm00SNWiC3Q==", - "dev": true, "requires": { "acorn": "^8.1.0", "acorn-walk": "^8.0.2" @@ -19441,8 +19419,7 @@ "acorn-walk": { "version": "8.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-8.2.0.tgz", - "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==", - "dev": true + "integrity": "sha512-k+iyHEuPgSw6SbuDpGQM+06HQUa04DZ3o+F6CSzXMvvI5KMvnaEqXe+YVe555R9nn6GPt404fos4wcgpw12SDA==" }, "actor-cheerio-scraper": { "version": "file:packages/actor-scraper/cheerio-scraper", @@ -19640,6 +19617,13 @@ } } }, + "apify-extra": { + "version": "file:packages/apify-extra", + "requires": { + "@types/bluebird": "^3.5.37", + "bluebird": "^3.7.2" + } + }, "aproba": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/aproba/-/aproba-2.0.0.tgz", @@ -20315,6 +20299,7 @@ "version": "7.0.4", "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.0", @@ -20818,8 +20803,7 @@ "decimal.js": { "version": "10.4.2", "resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.2.tgz", - "integrity": "sha512-ic1yEvwT6GuvaYwBLLY6/aFFgjZdySKTE8en/fkU3QICTmRtgtSlFn0u0BXN06InZwtfCelR7j8LRiDI/02iGA==", - "dev": true + "integrity": "sha512-ic1yEvwT6GuvaYwBLLY6/aFFgjZdySKTE8en/fkU3QICTmRtgtSlFn0u0BXN06InZwtfCelR7j8LRiDI/02iGA==" }, "decompress-response": { "version": "6.0.0", @@ -20850,7 +20834,6 @@ "version": "1.0.4", "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.4.tgz", "integrity": "sha512-eFuaLoy/Rxalv2kr+lqMlUnrDWV+3j4pljOIJgLIhI058IQfWJ7vXhyEIHu+HtC738klGALYxOKDO0bQP3tg8A==", - "dev": true, "requires": { "clone": "^1.0.2" } @@ -21039,8 +21022,7 @@ "eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", - "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", - "dev": true + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==" }, "ejs": { "version": "3.1.8", @@ -25371,8 +25353,7 @@ "p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", - "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", - "dev": true + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, "p-waterfall": { "version": "2.1.1", @@ -25508,8 +25489,7 @@ "path-exists": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", - "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", - "dev": true + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" }, "path-is-absolute": { "version": "1.0.1", @@ -25579,7 +25559,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", - "dev": true, "requires": { "find-up": "^4.0.0" }, @@ -25588,7 +25567,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", - "dev": true, "requires": { "locate-path": "^5.0.0", "path-exists": "^4.0.0" @@ -25598,7 +25576,6 @@ "version": "5.0.0", "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dev": true, "requires": { "p-locate": "^4.1.0" } @@ -25607,7 +25584,6 @@ "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dev": true, "requires": { "p-try": "^2.0.0" } @@ -25616,7 +25592,6 @@ "version": "4.1.0", "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dev": true, "requires": { "p-limit": "^2.2.0" } @@ -26299,7 +26274,6 @@ "version": "7.5.7", "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.5.7.tgz", "integrity": "sha512-z9MzKh/UcOqB3i20H6rtrlaE/CgjLOvheWK/9ILrbhROGTweAi1BaFsTT9FbwZi5Trr1qNRs+MXkhmR06awzQA==", - "dev": true, "requires": { "tslib": "^2.1.0" } @@ -27679,7 +27653,6 @@ "version": "8.0.1", "resolved": "https://registry.npmjs.org/cliui/-/cliui-8.0.1.tgz", "integrity": "sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==", - "dev": true, "requires": { "string-width": "^4.2.0", "strip-ansi": "^6.0.1", diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts index 0e456918c8..1544af0cd5 100644 --- a/packages/apify-extra/src/dataset.ts +++ b/packages/apify-extra/src/dataset.ts @@ -1,6 +1,6 @@ +/* eslint-disable max-classes-per-file */ import { Actor, Dataset as OriginalDataset } from 'apify'; import { log } from 'crawlee'; -import type { DatasetInfo } from '@crawlee/types'; import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const'; @@ -27,7 +27,7 @@ export interface ParallelPersistedPushDataOptions { idempotencyKey?: string; } -async function waitForCompletion(promises: (() => Promise)[], maxConcurrency: number): Promise { +export async function waitForCompletion(promises: (() => Promise)[], maxConcurrency: number): Promise { async function worker() { let job; /* eslint-disable-next-line no-cond-assign */ @@ -37,6 +37,28 @@ async function waitForCompletion(promises: (() => Promise)[], maxConcurren await Promise.all([...new Array(maxConcurrency)].map(() => worker())); } +export class ChunkTracker { + private readonly chunks: Record = {}; + + public add(chunkId: string): void { + this.chunks[chunkId] = true; + } + + public has(chunkId: string): boolean { + return this.chunks[chunkId] === true; + } + + public get(): string[] { + return Object.keys(this.chunks); + } + + constructor(data?: Record | null) { + if (data) { + Object.assign(this.chunks, data); + } + } +} + export class Dataset extends OriginalDataset { /** * Returns batches of items to be pushed to dataset. @@ -104,293 +126,39 @@ export class Dataset extends OriginalDataset { } }; - public async forEachParallel(func: (item: DatasetItem) => void, options: any = {}) { + public async forEachParallel( + func: Parameters[0], + options: Parameters[1] & { parallelLoads?: number; batchSize?: number; persistState: boolean }, + ) { const { parallelLoads = 20, batchSize = 50000 } = options; + const { offset: globalOffset = 0, limit: globalLimit = 0 } = options; - const { itemCount } = await this.getInfo() ?? { itemCount: 0 }; - - return waitForCompletion( - [...new Array(Math.ceil(itemCount / batchSize))] - .map((_, i) => () => this.forEach(func, { limit: batchSize, offset: batchSize * i })), - parallelLoads); - } - - // /** - // * Loads items from one or many datasets in parallel by chunking the items from each dataset into batches, - // * retaining order of both items and datasets. Useful for large loads. - // * By default returns one array of items in order of datasets provided. - // * By changing concatItems or concatDatasets options, you can get array of arrays (of arrays) back - // * Requires bluebird dependency and copy calculateLocalOffsetLimit function!!! - // * - // * @param {string[]} datasetIds IDs or names of datasets you want to load - // * @param {object} options Options with default values. - // * If both concatItems and concatDatasets are false, output of this function is an array of datasets containing arrays - // * of batches containig array of items. - // * concatItems concats all batches of one dataset into one array of items. - // * concatDatasets concat all datasets into one array of batches - // * Using both concatItems and concatDatasets gives you back a sinlge array of all items in order. - // * Both are true by default. - // * Data are not returned by fed to the supplied async function on the fly (reduces memory usage) - // * Will not load batches that were already processed before migration, does nothing if processFn is not used. - // * It does not persist the state inside processFn, that is a responsibillity of the caller (if needed) - // * You must not manipulate input parameters (and underlying datasets) between migrations or this will break - // */ - // public static async getDataParallel( - // datasetIds: string[], - // options: LoadDatasetItemsInParallelOptions = {}, - // ): Promise { - // const { - // processFn, - // parallelLoads = 20, - // batchSize = 50000, - // offset = 0, - // limit = 999999999, - // concatItems = true, - // concatDatasets = true, - // debugLog = false, - // persistLoadingStateForProcesFn = false, - // fields, - // // Figure out better name since this is useful for datasets by name on platform - // loadFromLocalDataset = false, - // } = options; - - // const LOG_PREFIX = `${APIFY_EXTRA_LOG_PREFIX}[loadDatasetItemsInParallel]:`; - - // if (!Actor.isAtHome() && loadFromLocalDataset && fields) { - // log.warning(`${LOG_PREFIX} fields option does not work on local datasets`); - // } - - // const client = Actor.apifyClient; - // const loadStart = Date.now(); - - // // If we use processFnLoadingState, we skip requests that are done - // const createRequestArray = async (processFnLoadingState: any) => { - // // We increment for each dataset so we remember their order - // let datasetIndex = 0; - - // // This array will be used to create promises to run in parallel - // const requestInfoArr = []; - - // for (const datasetId of datasetIds) { - // if (processFnLoadingState && !processFnLoadingState[datasetId]) { - // processFnLoadingState[datasetId] = {}; - // } - // // We get the number of items first and then we precreate request info objects - // let datasetInfo: DatasetInfo | undefined; - // if (loadFromLocalDataset) { - // const dataset = await Actor.openDataset(datasetId); - // datasetInfo = await dataset.getInfo(); - // } else { - // datasetInfo = await client.dataset(datasetId).get(); - // } - // if (!datasetInfo) { - // throw new Error(`${LOG_PREFIX} Dataset ${datasetId} was not found`); - // } - // const { itemCount } = datasetInfo; - // if (debugLog) { - // log.info(`Dataset ${datasetId} has ${itemCount} items`); - // } - // const numberOfBatches = Math.ceil(itemCount / batchSize); - - // for (let i = 0; i < numberOfBatches; i++) { - // const localOffsetLimit = calculateLocalOffsetLimit({ offset, limit, localStart: i * batchSize, batchSize }); - // if (!localOffsetLimit) { - // continue; - // } - - // if (processFnLoadingState) { - // if (!processFnLoadingState[datasetId][localOffsetLimit.offset]) { - // processFnLoadingState[datasetId][localOffsetLimit.offset] = { done: false }; - // } else if (processFnLoadingState[datasetId][localOffsetLimit.offset].done) { - // if (debugLog) { - // log.info(`Batch for dataset ${datasetId}, offset: ${localOffsetLimit.offset} was already processed, skipping...`); - // } - // continue; - // } - // } - - // requestInfoArr.push({ - // index: i, - // offset: localOffsetLimit.offset, - // limit: localOffsetLimit.limit, - // datasetId, - // datasetIndex, - // }); - // } - - // datasetIndex++; - // } - // return requestInfoArr; - // }; - - // // This is array of arrays. Top level array is for each dataset and inside one entry for each batch (in order) - // const loadedBatchedArr: DatasetItem[][][] = []; - - // let totalLoaded = 0; - // const totalLoadedPerDataset: Record = {}; - - // const processFnLoadingState: any = persistLoadingStateForProcesFn - // ? (await Actor.getValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY) || {}) - // : null; - - // Actor.on('persistState', async () => { - // await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); - // }); + const chunkTrackerName = `${APIFY_EXTRA_KV_RECORD_PREFIX}CHUNKS${this.name}`; + const chunkTracker = new ChunkTracker(await Actor.getValue>(chunkTrackerName)); - // const requestInfoArr = await createRequestArray(processFnLoadingState); - // if (debugLog) { - // log.info(`Number of requests to do: ${requestInfoArr.length}`); - // } - - // // Now we execute all the requests in parallel (with defined concurrency) - // await concurrentWait(requestInfoArr.map((requestInfoObj) => async () => { - // const { index, datasetId, datasetIndex } = requestInfoObj; - - // const getDataOptions = { - // offset: requestInfoObj.offset, - // limit: requestInfoObj.limit, - // fields, - // }; - // let datasetResult; - // if (loadFromLocalDataset) { - // // This open should be cached - // const dataset = await Actor.openDataset(datasetId); - - // if (!Actor.isAtHome()) { - // delete getDataOptions.fields; - // } - // datasetResult = await dataset.getData(getDataOptions); - // } else { - // datasetResult = await client.dataset(datasetId).listItems(getDataOptions); - // } - - // const { items } = datasetResult; - - // if (!totalLoadedPerDataset[datasetId]) { - // totalLoadedPerDataset[datasetId] = 0; - // } - - // totalLoadedPerDataset[datasetId] += items.length; - // totalLoaded += items.length; - - // if (debugLog) { - // log.info( - // `Items loaded from dataset ${datasetId}: ${items.length}, offset: ${requestInfoObj.offset}, - // total loaded from dataset ${datasetId}: ${totalLoadedPerDataset[datasetId]}, - // total loaded: ${totalLoaded}`, - // ); - // } - // // We either collect the data or we process them on the fly - // if (processFn) { - // await processFn(items, { datasetId, datasetOffset: requestInfoObj.offset }); - // if (processFnLoadingState) { - // processFnLoadingState[datasetId][requestInfoObj.offset].done = true; - // } - // } else { - // if (!loadedBatchedArr[datasetIndex]) { - // loadedBatchedArr[datasetIndex] = []; - // } - // // Now we correctly assign the items into the main array - // loadedBatchedArr[datasetIndex][index] = items; - // } - // }), parallelLoads); - - // if (debugLog) { - // log.info(`Loading took ${Math.round((Date.now() - loadStart) / 1000)} seconds`); - // } - - // if (processFnLoadingState) { - // await Actor.setValue(PROCESS_FN_LOADING_STATE_KV_RECORD_KEY, processFnLoadingState); - // } - - // if (processFn) { - // return undefined; - // } - - // let resultItems: DatasetItem[] | DatasetItem[][] | DatasetItem[][][] = loadedBatchedArr; - // if (concatItems) { - // for (let i = 0; i < loadedBatchedArr.length; i++) { - // resultItems[i] = loadedBatchedArr[i].flatMap((item) => item); - // } - // } - -// if (concatDatasets) { -// resultItems = loadedBatchedArr.flatMap((item) => item); -// } -// return loadedBatchedArr; -// }; -} - -interface CalculateLocalOffsetLimitParams { - offset: number; - limit: number; - localStart: number; - batchSize: number; -} - -type CalculateLocalOffsetLimitResult = - null | - { offset: number; limit: number }; + let isMigrating = false; + const migrationCallback = async () => { + isMigrating = true; + await Actor.setValue(chunkTrackerName, chunkTracker.get()); + }; -// Returns either null if offset/limit does not fit the current chunk -// or { offset, limit } object -const calculateLocalOffsetLimit = ({ offset, limit, localStart, batchSize }: CalculateLocalOffsetLimitParams): CalculateLocalOffsetLimitResult => { - const localEnd = localStart + batchSize; - const inputEnd = offset + limit; + Actor.on('migrating', migrationCallback); + Actor.on('aborting', migrationCallback); - // Offset starts after the current chunk - if (offset >= localEnd) { - return null; - } - // Offset + limit ends before our chunk - if (inputEnd <= localStart) { - return null; - } + Actor.on('persistState', async () => { + await Actor.setValue(chunkTrackerName, chunkTracker.get()); + }); - // Now we know that the some data are in the current batch - const calculateLimit = () => { - // limit overflows current batch - if (inputEnd >= localEnd) { - // Now either the offset is less than local start and we do whole batch - if (offset < localStart) { - return batchSize; - } - // Or it is inside the current batch and we slice it from the start (including whole batch) - return localEnd - offset; - // eslint-disable-next-line no-else-return - } else { // Consider (inputEnd < localEnd) Means limit ends inside current batch - if (offset < localStart) { - return inputEnd - localStart; - } - // This means both offset and limit are inside current batch - return inputEnd - offset; - } - }; + const { itemCount } = await this.getInfo() ?? { itemCount: 0 }; - return { - offset: Math.max(localStart, offset), - limit: calculateLimit(), + return waitForCompletion( + [...new Array(Math.ceil((itemCount < globalLimit ? itemCount : globalLimit) / batchSize))] + .filter((_, i) => !chunkTracker.has(`${globalOffset + i * batchSize}`)) + .map((_, i) => async () => { + if (isMigrating) await new Promise(() => {}); // blocks indefinitely - after a while, stops the entire execution + await this.forEach(func, { ...options, limit: batchSize, offset: globalOffset + batchSize * i }); + chunkTracker.add(`${globalOffset + batchSize * i}`); + }), + parallelLoads); }; -}; - -const PROCESS_FN_LOADING_STATE_KV_RECORD_KEY = `${APIFY_EXTRA_KV_RECORD_PREFIX}PROCESS-FN-LOADING-STATE`; - -export interface LoadDatasetItemsInParallelOptions { - parallelLoads?: number; - batchSize?: number; - offset?: number; - limit?: number; - concatItems?: boolean; - concatDatasets?: boolean; - debugLog?: boolean; - persistLoadingStateForProcesFn?: boolean; - fields?: string[]; - // If run outside of Apify platform, will fetch local datasets instead. On Apify platform, this is ignored. - loadFromLocalDataset?: boolean; - processFn?: (items: DatasetItem[], params: { datasetId: string; datasetOffset: number }) => Promise; } - -export type LoadDatasetItemsInParallelResult = - undefined | - DatasetItem[] | - DatasetItem[][]; diff --git a/test/apify/extra-dataset.test.ts b/test/apify/extra-dataset.test.ts new file mode 100644 index 0000000000..3f76496034 --- /dev/null +++ b/test/apify/extra-dataset.test.ts @@ -0,0 +1,62 @@ +import { ENV_VARS } from '@apify/consts'; +import { Actor, Configuration, PlatformEventManager, log } from 'apify'; +import { Server } from 'ws'; +import { Dataset } from 'apify-extra'; + +describe('apify-extra dataset', () => { + let wss: Server = null; + const config = Configuration.getGlobalConfig(); + const events = new PlatformEventManager(config); + config.useEventManager(events); + + beforeEach(() => { + wss = new Server({ port: 9099 }); + process.env[ENV_VARS.ACTOR_EVENTS_WS_URL] = 'ws://localhost:9099/someRunId'; + process.env[ENV_VARS.TOKEN] = 'dummy'; + }); + afterEach((done) => { + delete process.env[ENV_VARS.ACTOR_EVENTS_WS_URL]; + delete process.env[ENV_VARS.TOKEN]; + wss.close(done); + }); + + test('forEachParallel', async () => { + // let wsClosed = false; + // const isWsConnected = new Promise((resolve) => { + // wss.on('connection', (ws, req) => { + // ws.on('close', () => { + // wsClosed = true; + // }); + // resolve(ws); + // expect(req.url).toBe('/someRunId'); + // const send = (obj: Dictionary) => ws.send(JSON.stringify(obj)); + + // setTimeout(() => send({ name: 'migrating' }), 10); + // }); + // }); + + const forEachSpy = jest.spyOn(Dataset.prototype, 'forEach').mockImplementation(); + jest.spyOn(Dataset.prototype, 'getInfo').mockImplementation(async () => ({ + itemCount: 20, + } as any)); + jest.spyOn(Actor.prototype, 'getValue').mockImplementation(async () => ({})); + + // await isWsConnected; + + const dataset = new Dataset({ + client: Configuration.getStorageClient(), + id: 'dataset-forEachParallel-test', + }); + + await dataset.forEachParallel(() => { log.debug('noop'); }, { persistState: true }); + + // // Cleanup. + // await new Promise((resolve) => { + // wss.close(async () => { + // await sleep(10); // Here must be short sleep to get following line to later tick + // expect(wsClosed).toBe(true); + // resolve(); + // }); + // }); + }, 60e3); +}); From 41d9385ac256e527e5d3d1e4e3c1880f377d6454 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 14 Nov 2022 13:19:38 +0100 Subject: [PATCH 7/9] feat: parallel funcs fix, e2e + stub unit tests --- packages/apify-extra/src/dataset.ts | 188 +++++++++++++++------------- test/apify/extra-dataset.test.ts | 45 +++---- test/e2e/extra-dataset/test.mjs | 29 +++++ 3 files changed, 149 insertions(+), 113 deletions(-) create mode 100644 test/e2e/extra-dataset/test.mjs diff --git a/packages/apify-extra/src/dataset.ts b/packages/apify-extra/src/dataset.ts index 1544af0cd5..36710d0194 100644 --- a/packages/apify-extra/src/dataset.ts +++ b/packages/apify-extra/src/dataset.ts @@ -6,27 +6,6 @@ import { APIFY_EXTRA_KV_RECORD_PREFIX, APIFY_EXTRA_LOG_PREFIX } from './const'; export type DatasetItem = Exclude[0], any[]>; -export interface ParallelPersistedPushDataOptions { - /** - * Number of items to be pushed in one push call. - * Should not be higher than 1000 to ensure each push call finishes when migration happens. - * - * @default 1000 - */ - uploadBatchSize?: number; - /** - * Number of push calls to be done in parallel. Apify API should handle up to 30 parallel requests but better be careful. - * - * @default 10 - */ - parallelPushes?: number; - /** - * You must provide idempotency key if you want to call this function multiple times in the same run. - * The key should only contain letters and numbers. - */ - idempotencyKey?: string; -} - export async function waitForCompletion(promises: (() => Promise)[], maxConcurrency: number): Promise { async function worker() { let job; @@ -61,103 +40,144 @@ export class ChunkTracker { export class Dataset extends OriginalDataset { /** - * Returns batches of items to be pushed to dataset. - * @param data Items to be pushed to dataset. - * @param batchSize Number of items to be pushed in one push call. - * @param batchCount Number of batches to be returned. - * @returns Array of batches of items to be pushed to dataset. - */ - private getBatches(data: DatasetItem | DatasetItem[], batchSize: number, batchCount: number) { - if (!Array.isArray(data)) { - return [[data]]; - } - return [...Array(batchCount).keys()] - .map((x: number) => data.slice(x * batchSize, (x + 1) * batchSize)) - .filter((x) => x.length !== 0); - } - - /** - * Push data to dataset in parallel while also resuming correctly after Actor migration. - * This should be used when you want to push a lot of items to dataset at once. - * If you call this function multiple times in the same run, - * you must provide different idempotencyKey option for each call. + * Stores an object or an array of objects to the dataset. + * The function returns a promise that resolves when the operation finishes. + * It has no result, but throws on invalid args or other errors. + * + * **IMPORTANT**: Make sure to use the `await` keyword when calling `pushDataParallel()`, + * otherwise the crawler process might finish before the data is stored! + * + * The size of the data is limited by the receiving API and therefore `pushDataParallel()` will only + * allow objects whose JSON representation is smaller than 9MB. When an array is passed, + * none of the included objects + * may be larger than 9MB, but the array itself may be of any size. + * + * This method parallellizes the pushData calls to the Apify API, which can handle up to 30 parallel requests. + * It also ensures keeps track of the progress and can resume the push if the actor is migrated. + * Unline the `pushData` method, this method does not guarantee the order of items. + * + * @param data Object or array of objects containing data to be stored in the dataset. + * The objects must be serializable to JSON and the JSON representation of each object must be smaller than 9MB. + * @param [options] All `pushDataParallel()` parameters. + * @param [options.batchSize] Number of items to be pushed in one push call. + * Should not be higher than 1000 to ensure each push call finishes when migration happens. + * @param [options.parallelPushes] Number of push calls to be done in parallel. Apify API should handle up to 30 parallel requests. + * @param [options.idempotencyKey] By providing different idempotency keys (any string), you can call this function multiple times in the same run. */ - public async pushDataParallel(data: DatasetItem | DatasetItem[], options: ParallelPersistedPushDataOptions = {}) { + public async pushDataParallel(data: DatasetItem | DatasetItem[], options: { + batchSize?: number; + parallelPushes?: number; + idempotencyKey?: string; + } = {}) { if (!Array.isArray(data)) { return this.pushData(data); } const { - uploadBatchSize = 1000, + batchSize = 1000, parallelPushes = 10, idempotencyKey = '', } = options; - let isMigrating = false; - Actor.on('migrating', () => { isMigrating = true; }); - Actor.on('aborting', () => { isMigrating = true; }); + if (parallelPushes > 30) { + log.warning(`${APIFY_EXTRA_LOG_PREFIX} Setting the parallelPushes option larger than 30 can lead to problems with the Apify Platform API.`); + } const sanitizedIdempotencyKey = idempotencyKey.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 30); + const chunkTrackerName = `${APIFY_EXTRA_KV_RECORD_PREFIX}-PUSH-${this.id}-${sanitizedIdempotencyKey}`; + const chunkTracker = new ChunkTracker(await Actor.getValue>(chunkTrackerName)); - const kvRecordName = `${APIFY_EXTRA_KV_RECORD_PREFIX}STATE-PUSHED-COUNT${this.name}${sanitizedIdempotencyKey}`; - let pushedItemsCount = await Actor.getValue(kvRecordName) ?? 0; - - Actor.on('persistState', async () => { - await Actor.setValue(kvRecordName, pushedItemsCount); - }); - - const stepSize = uploadBatchSize * parallelPushes; - - for (let i = pushedItemsCount; i < data.length; i += stepSize) { - if (isMigrating) { - log.info(`${APIFY_EXTRA_LOG_PREFIX}[pushDataParallel]: Stopping push because of migration`); - // hang indefinitely until migration is done - await new Promise(() => {}); - } + let isMigrating = false; + const migrationCallback = async (migrating: boolean) => { + isMigrating = migrating ?? true; + await Actor.setValue(chunkTrackerName, chunkTracker.get()); + }; - const itemsToPush = data.slice(i, i + stepSize); - const batches = this.getBatches(itemsToPush, uploadBatchSize, parallelPushes); - const pushPromises = batches.map(this.pushData); + Actor.on('migrating', migrationCallback); + Actor.on('aborting', migrationCallback); + Actor.on('persistState', () => migrationCallback(false)); - // We must update it before awaiting the promises because the push can take time - // and migration can cut us off but the items will already be on the way to dataset - pushedItemsCount += itemsToPush.length; - await Promise.all(pushPromises); - } + return waitForCompletion( + [...new Array(Math.ceil(data.length / batchSize))] + .filter((_, i) => !chunkTracker.has(`${batchSize * i}`)) + .map((_, i) => async () => { + if (isMigrating) { + log.info(`${APIFY_EXTRA_LOG_PREFIX}[pushParallel]: Stopping pushParallel because of migration`); + await new Promise(() => {}); + } + + chunkTracker.add(`${batchSize * i}`); + const currentSlice = data.slice(batchSize * i, batchSize * (i + 1)); + if (currentSlice.length > 0) { + await this.pushData(currentSlice); + } + }), + parallelPushes); }; + /** + * Iterates over dataset items, passing every item to the provided `func` function. + * Each invocation of `func` is called with two arguments: `(item, index)`. Index specifies the zero-based index of the item in the dataset. + * + * If the `func` function returns a Promise, it is awaited. + * If it throws an error, the iteration is aborted and the `forEachParallel` function throws the error. + * + * **Example usage** + * ```typescript + * const dataset = await Dataset.open('my-results'); + * await dataset.forEachParallel(async (item, index) => { + * console.log(`Item at ${index}: ${JSON.stringify(item)}`); + * }); + * ``` + * + * *Important note*: Unlike the `forEach` method, this method processes items in parallel and does not guarantee the order of items. + * It also doesn't wait before calling the provided function for the next item. + * + * @param func A function that is called for every item in the dataset. + * @param [options] All `forEach()` parameters. + * @param [options.parallelLoads] Maximum number of item batches to be processed in parallel. + * @param [options.batchSize] Maximum number of items to be processed in one batch. + * @param [options.persistState] If `true`, the processing state will be persisted between actor migrations and runs. + * @returns {Promise} + */ public async forEachParallel( func: Parameters[0], - options: Parameters[1] & { parallelLoads?: number; batchSize?: number; persistState: boolean }, + options: Parameters[1] & { parallelLoads?: number; batchSize?: number; persistState: boolean; idempotencyKey?: string }, ) { - const { parallelLoads = 20, batchSize = 50000 } = options; - const { offset: globalOffset = 0, limit: globalLimit = 0 } = options; + const { parallelLoads = 20, batchSize = 50000, persistState, idempotencyKey = '' } = options; + const { offset: globalOffset = 0, limit: globalLimit = Infinity } = options; - const chunkTrackerName = `${APIFY_EXTRA_KV_RECORD_PREFIX}CHUNKS${this.name}`; - const chunkTracker = new ChunkTracker(await Actor.getValue>(chunkTrackerName)); + const sanitizedIdempotencyKey = idempotencyKey.replace(/[^a-zA-Z0-9]/g, '-').slice(0, 30); + const chunkTrackerName = `${APIFY_EXTRA_KV_RECORD_PREFIX}FOREACH-${this.id}-${sanitizedIdempotencyKey}`; + const chunkTracker = new ChunkTracker(persistState ? await Actor.getValue>(chunkTrackerName) : undefined); let isMigrating = false; - const migrationCallback = async () => { - isMigrating = true; - await Actor.setValue(chunkTrackerName, chunkTracker.get()); + + const migrationCallback = async (migrating: boolean) => { + isMigrating = migrating ?? true; + await Actor.setValue(chunkTrackerName, persistState ? chunkTracker.get() : null); }; Actor.on('migrating', migrationCallback); Actor.on('aborting', migrationCallback); - - Actor.on('persistState', async () => { - await Actor.setValue(chunkTrackerName, chunkTracker.get()); - }); + Actor.on('persistState', () => migrationCallback(false)); const { itemCount } = await this.getInfo() ?? { itemCount: 0 }; return waitForCompletion( - [...new Array(Math.ceil((itemCount < globalLimit ? itemCount : globalLimit) / batchSize))] + [...new Array(Math.ceil((itemCount < globalLimit ? itemCount : globalLimit) / batchSize))] // every item represents one chunk .filter((_, i) => !chunkTracker.has(`${globalOffset + i * batchSize}`)) .map((_, i) => async () => { - if (isMigrating) await new Promise(() => {}); // blocks indefinitely - after a while, stops the entire execution - await this.forEach(func, { ...options, limit: batchSize, offset: globalOffset + batchSize * i }); + if (isMigrating) { + log.info(`${APIFY_EXTRA_LOG_PREFIX}[forEachParallel]: Stopping forEachParallel because of migration`); + // hang indefinitely until migration is done + await new Promise(() => {}); + } + + const { items } = await this.getData({ limit: batchSize, offset: globalOffset + batchSize * i }); + chunkTracker.add(`${globalOffset + batchSize * i}`); + await Promise.all(items.map((item, b) => func(item, globalOffset + batchSize * i + b))); }), parallelLoads); }; diff --git a/test/apify/extra-dataset.test.ts b/test/apify/extra-dataset.test.ts index 3f76496034..1b3f2a5a06 100644 --- a/test/apify/extra-dataset.test.ts +++ b/test/apify/extra-dataset.test.ts @@ -1,9 +1,9 @@ import { ENV_VARS } from '@apify/consts'; -import { Actor, Configuration, PlatformEventManager, log } from 'apify'; +import { Actor, Configuration, Dataset as DatasetClassic, PlatformEventManager, log } from 'apify'; import { Server } from 'ws'; import { Dataset } from 'apify-extra'; -describe('apify-extra dataset', () => { +describe('forEachParallel', () => { let wss: Server = null; const config = Configuration.getGlobalConfig(); const events = new PlatformEventManager(config); @@ -20,43 +20,30 @@ describe('apify-extra dataset', () => { wss.close(done); }); - test('forEachParallel', async () => { - // let wsClosed = false; - // const isWsConnected = new Promise((resolve) => { - // wss.on('connection', (ws, req) => { - // ws.on('close', () => { - // wsClosed = true; - // }); - // resolve(ws); - // expect(req.url).toBe('/someRunId'); - // const send = (obj: Dictionary) => ws.send(JSON.stringify(obj)); - - // setTimeout(() => send({ name: 'migrating' }), 10); - // }); - // }); - + test('slicing should work', async () => { const forEachSpy = jest.spyOn(Dataset.prototype, 'forEach').mockImplementation(); jest.spyOn(Dataset.prototype, 'getInfo').mockImplementation(async () => ({ - itemCount: 20, + itemCount: 229, // a prime number } as any)); jest.spyOn(Actor.prototype, 'getValue').mockImplementation(async () => ({})); - // await isWsConnected; - const dataset = new Dataset({ client: Configuration.getStorageClient(), id: 'dataset-forEachParallel-test', }); - await dataset.forEachParallel(() => { log.debug('noop'); }, { persistState: true }); + await dataset.forEachParallel(() => { log.debug('noop'); }, { persistState: true, batchSize: 7 }); - // // Cleanup. - // await new Promise((resolve) => { - // wss.close(async () => { - // await sleep(10); // Here must be short sleep to get following line to later tick - // expect(wsClosed).toBe(true); - // resolve(); - // }); - // }); + expect(forEachSpy).toBeCalledTimes(Math.ceil(229 / 7)); + + [...Array(Math.ceil(229 / 7))].forEach((_, i) => { + expect(forEachSpy).toBeCalledWith( + expect.any(Function), + expect.objectContaining({ + limit: 7, + offset: i * 7, + })); + }); + forEachSpy.mockRestore(); }, 60e3); }); diff --git a/test/e2e/extra-dataset/test.mjs b/test/e2e/extra-dataset/test.mjs new file mode 100644 index 0000000000..2844aaafa0 --- /dev/null +++ b/test/e2e/extra-dataset/test.mjs @@ -0,0 +1,29 @@ +import { getTestDir, initialize, expect } from '../tools.mjs'; +import { Dataset } from 'apify-extra'; +import { Actor, Configuration, log } from 'apify'; + +const testDir = getTestDir(import.meta.url); + +const exit = process.exit; +process.exit = () => {}; + +await initialize(testDir); +await Actor.openDataset('dataset-forEachParallel-test'); +const dataset = new Dataset({ + id: 'dataset-forEachParallel-test', + client: Configuration.getStorageClient(), +}); + +const ITEM_COUNT = 229; +const target = []; + +await dataset.pushDataParallel([...new Array(ITEM_COUNT)].map((_, i) => ({ index: i })), { batchSize: 10, parallelPushes: 4 }); +await dataset.forEachParallel((x) => target.push(x), { persistState: true, batchSize: 10 }); + +expect((await dataset.getData()).total === ITEM_COUNT, 'all items pushed'); +expect([...new Array(ITEM_COUNT)].every((_, i) => target.some(x => x.index === i)), 'all items processed'); + +expect(target.length === ITEM_COUNT, `forEach called ${target.length} times.`); + +process.exit = exit; +process.exit(0); From 1300fd8f680157511dc81f958f983fa8584b68b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 14 Nov 2022 13:24:53 +0100 Subject: [PATCH 8/9] chore: update lockfile --- package-lock.json | 34 +++++++++++++--------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/package-lock.json b/package-lock.json index e2803734f4..163f6fd5c7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1147,7 +1147,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.1.1.tgz", "integrity": "sha512-jdrEX1eEe7tFw5d3A32A5/Fk1iQXjWzXIAtxAxi4s5twHaWeo/YnXCEspKEMGu3kQ9hFQcR2CFzdA6dc5V1waA==", - "dev": true, "dependencies": { "@crawlee/templates": "^3.1.1", "ansi-colors": "^4.1.3", @@ -1219,7 +1218,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.1.1.tgz", "integrity": "sha512-l94vFSGjzpGUmW3GgQokxM+9CkSYWwQCQp2fbliB+nMJ5nr6cGTNuar5PtyPWKUE8fd0WmJPm+11ZwnBbSGDlw==", - "dev": true, "dependencies": { "@apify/utilities": "^2.1.4", "@crawlee/http": "^3.1.1", @@ -1310,7 +1308,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.1.1.tgz", "integrity": "sha512-pmRbxWpZpc+8XL8vB8uiTp3EMd+TvC/dVcmcd5GgaYLPRzgv2+g85Dh4YdgBQB0/LcYwkFgotPi/t/+M8cb3Cg==", - "dev": true, "dependencies": { "ansi-colors": "^4.1.3", "inquirer": "^9.0.0", @@ -4156,6 +4153,11 @@ "@babel/types": "^7.3.0" } }, + "node_modules/@types/bluebird": { + "version": "3.5.37", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.37.tgz", + "integrity": "sha512-g2qEd+zkfkTEudA2SrMAeAvY7CrFqtbsLILm2dT2VIeKTqMqVzcdfURlvu6FU3srRgbmXN1Srm94pg34EIehww==" + }, "node_modules/@types/content-type": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@types/content-type/-/content-type-1.1.5.tgz", @@ -4227,7 +4229,6 @@ "version": "20.0.1", "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz", "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==", - "dev": true, "dependencies": { "@types/node": "*", "@types/tough-cookie": "*", @@ -4612,7 +4613,6 @@ "version": "8.8.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", - "dev": true, "bin": { "acorn": "bin/acorn" }, @@ -6199,7 +6199,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.1.1.tgz", "integrity": "sha512-yi4pIkmLoqPzcP1zHOT1uI6TKdtD5kflXUVW1fAI1iKl1bBjMidF3I4+ZWA/QWZRNCxe7BZbJkVsq1Zxc6A9uA==", - "dev": true, "dependencies": { "@crawlee/basic": "^3.1.1", "@crawlee/browser": "^3.1.1", @@ -8968,7 +8967,6 @@ "version": "8.2.5", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.5.tgz", "integrity": "sha512-QAgPDQMEgrDssk1XiwwHoOGYF9BAbUcc1+j+FhEvaOt8/cKRqyLn0U5qA6F74fGhTMGxf92pOvPBeh29jQJDTQ==", - "dev": true, "dependencies": { "ansi-escapes": "^4.2.1", "chalk": "^4.1.1", @@ -10126,7 +10124,6 @@ "version": "20.0.2", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.2.tgz", "integrity": "sha512-AHWa+QO/cgRg4N+DsmHg1Y7xnz+8KU3EflM0LVDTdmrYOc1WWTSkOjtpUveQH+1Bqd5rtcVnb/DuxV/UjDO4rA==", - "dev": true, "dependencies": { "abab": "^2.0.6", "acorn": "^8.8.0", @@ -10171,7 +10168,6 @@ "version": "8.11.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", - "dev": true, "engines": { "node": ">=10.0.0" }, @@ -15561,7 +15557,6 @@ "version": "17.6.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", @@ -15735,6 +15730,7 @@ "license": "Apache-2.0", "dependencies": { "@types/bluebird": "^3.5.37", + "apify": "^3.0.0", "bluebird": "^3.7.2" }, "engines": { @@ -16638,7 +16634,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/cli/-/cli-3.1.1.tgz", "integrity": "sha512-jdrEX1eEe7tFw5d3A32A5/Fk1iQXjWzXIAtxAxi4s5twHaWeo/YnXCEspKEMGu3kQ9hFQcR2CFzdA6dc5V1waA==", - "dev": true, "requires": { "@crawlee/templates": "^3.1.1", "ansi-colors": "^4.1.3", @@ -16698,7 +16693,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/jsdom/-/jsdom-3.1.1.tgz", "integrity": "sha512-l94vFSGjzpGUmW3GgQokxM+9CkSYWwQCQp2fbliB+nMJ5nr6cGTNuar5PtyPWKUE8fd0WmJPm+11ZwnBbSGDlw==", - "dev": true, "requires": { "@apify/utilities": "^2.1.4", "@crawlee/http": "^3.1.1", @@ -16761,7 +16755,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/@crawlee/templates/-/templates-3.1.1.tgz", "integrity": "sha512-pmRbxWpZpc+8XL8vB8uiTp3EMd+TvC/dVcmcd5GgaYLPRzgv2+g85Dh4YdgBQB0/LcYwkFgotPi/t/+M8cb3Cg==", - "dev": true, "requires": { "ansi-colors": "^4.1.3", "inquirer": "^9.0.0", @@ -19042,6 +19035,11 @@ "@babel/types": "^7.3.0" } }, + "@types/bluebird": { + "version": "3.5.37", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.37.tgz", + "integrity": "sha512-g2qEd+zkfkTEudA2SrMAeAvY7CrFqtbsLILm2dT2VIeKTqMqVzcdfURlvu6FU3srRgbmXN1Srm94pg34EIehww==" + }, "@types/content-type": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/@types/content-type/-/content-type-1.1.5.tgz", @@ -19113,7 +19111,6 @@ "version": "20.0.1", "resolved": "https://registry.npmjs.org/@types/jsdom/-/jsdom-20.0.1.tgz", "integrity": "sha512-d0r18sZPmMQr1eG35u12FZfhIXNrnsPU/g5wvRKCUf/tOGilKKwYMYGqh33BNR6ba+2gkHw1EUiHoN3mn7E5IQ==", - "dev": true, "requires": { "@types/node": "*", "@types/tough-cookie": "*", @@ -19397,8 +19394,7 @@ "acorn": { "version": "8.8.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.1.tgz", - "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==", - "dev": true + "integrity": "sha512-7zFpHzhnqYKrkYdUjF1HI1bzd0VygEGX8lFk4k5zVMqHEoES+P+7TKI+EvLO9WVMJ8eekdO0aDEK044xTXwPPA==" }, "acorn-globals": { "version": "7.0.1", @@ -19621,6 +19617,7 @@ "version": "file:packages/apify-extra", "requires": { "@types/bluebird": "^3.5.37", + "apify": "^3.0.0", "bluebird": "^3.7.2" } }, @@ -20652,7 +20649,6 @@ "version": "3.1.1", "resolved": "https://registry.npmjs.org/crawlee/-/crawlee-3.1.1.tgz", "integrity": "sha512-yi4pIkmLoqPzcP1zHOT1uI6TKdtD5kflXUVW1fAI1iKl1bBjMidF3I4+ZWA/QWZRNCxe7BZbJkVsq1Zxc6A9uA==", - "dev": true, "requires": { "@crawlee/basic": "^3.1.1", "@crawlee/browser": "^3.1.1", @@ -22721,7 +22717,6 @@ "version": "8.2.5", "resolved": "https://registry.npmjs.org/inquirer/-/inquirer-8.2.5.tgz", "integrity": "sha512-QAgPDQMEgrDssk1XiwwHoOGYF9BAbUcc1+j+FhEvaOt8/cKRqyLn0U5qA6F74fGhTMGxf92pOvPBeh29jQJDTQ==", - "dev": true, "requires": { "ansi-escapes": "^4.2.1", "chalk": "^4.1.1", @@ -23591,7 +23586,6 @@ "version": "20.0.2", "resolved": "https://registry.npmjs.org/jsdom/-/jsdom-20.0.2.tgz", "integrity": "sha512-AHWa+QO/cgRg4N+DsmHg1Y7xnz+8KU3EflM0LVDTdmrYOc1WWTSkOjtpUveQH+1Bqd5rtcVnb/DuxV/UjDO4rA==", - "dev": true, "requires": { "abab": "^2.0.6", "acorn": "^8.8.0", @@ -23625,7 +23619,6 @@ "version": "8.11.0", "resolved": "https://registry.npmjs.org/ws/-/ws-8.11.0.tgz", "integrity": "sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg==", - "dev": true, "requires": {} } } @@ -27638,7 +27631,6 @@ "version": "17.6.2", "resolved": "https://registry.npmjs.org/yargs/-/yargs-17.6.2.tgz", "integrity": "sha512-1/9UrdHjDZc0eOU0HxOHoS78C69UD3JRMvzlJ7S79S2nTaWRA/whGCTV8o9e/N/1Va9YIV7Q4sOxD8VV4pCWOw==", - "dev": true, "requires": { "cliui": "^8.0.1", "escalade": "^3.1.1", From 08324951baf823f33ec0d1b472214dff1b338a76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jind=C5=99ich=20B=C3=A4r?= Date: Mon, 14 Nov 2022 14:06:07 +0100 Subject: [PATCH 9/9] fix: test fix --- test/apify/extra-dataset.test.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/test/apify/extra-dataset.test.ts b/test/apify/extra-dataset.test.ts index 1b3f2a5a06..f551cffc33 100644 --- a/test/apify/extra-dataset.test.ts +++ b/test/apify/extra-dataset.test.ts @@ -1,5 +1,5 @@ import { ENV_VARS } from '@apify/consts'; -import { Actor, Configuration, Dataset as DatasetClassic, PlatformEventManager, log } from 'apify'; +import { Actor, Configuration, PlatformEventManager, log } from 'apify'; import { Server } from 'ws'; import { Dataset } from 'apify-extra'; @@ -21,7 +21,9 @@ describe('forEachParallel', () => { }); test('slicing should work', async () => { - const forEachSpy = jest.spyOn(Dataset.prototype, 'forEach').mockImplementation(); + const getDataSpy = jest.spyOn(Dataset.prototype, 'getData').mockImplementation(async ({ limit }) => ( + { items: [...new Array(limit).keys()] } as any + )); jest.spyOn(Dataset.prototype, 'getInfo').mockImplementation(async () => ({ itemCount: 229, // a prime number } as any)); @@ -32,18 +34,17 @@ describe('forEachParallel', () => { id: 'dataset-forEachParallel-test', }); - await dataset.forEachParallel(() => { log.debug('noop'); }, { persistState: true, batchSize: 7 }); + await dataset.forEachParallel((x) => { log.debug(`processing ${x}`); }, { persistState: true, batchSize: 7 }); - expect(forEachSpy).toBeCalledTimes(Math.ceil(229 / 7)); + expect(getDataSpy).toBeCalledTimes(Math.ceil(229 / 7)); [...Array(Math.ceil(229 / 7))].forEach((_, i) => { - expect(forEachSpy).toBeCalledWith( - expect.any(Function), + expect(getDataSpy).toBeCalledWith( expect.objectContaining({ limit: 7, offset: i * 7, })); }); - forEachSpy.mockRestore(); + getDataSpy.mockRestore(); }, 60e3); });