From 79d29a2101cc1ce0d63d5b59817c7f6e53438002 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 2 Apr 2024 18:52:49 +0200 Subject: [PATCH 01/42] add csv format --- index.js | 6 +-- lib/formats/csv.js | 70 +++++++++++++++++++++++++++++++++++ lib/formats/gzip.js | 15 +++++++- lib/formats/index.js | 7 ++++ lib/methods/export.js | 85 ++++++++++++++++++++++++++----------------- package.json | 4 +- 6 files changed, 148 insertions(+), 39 deletions(-) create mode 100644 lib/formats/csv.js create mode 100644 lib/formats/index.js diff --git a/index.js b/index.js index d0e37171..cb72d795 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,7 @@ const fs = require('fs'); const path = require('path'); const methods = require('./lib/methods'); const apiRoutes = require('./lib/apiRoutes'); -const gzip = require('./lib/formats/gzip'); +const formats = require('./lib/formats'); module.exports = { bundle: { @@ -21,8 +21,8 @@ module.exports = { }, init(self) { self.formats = { - gzip, - ...(self.options.formats || {}) + ...formats, + ...self.options.formats || {} }; self.enableBrowserData(); diff --git a/lib/formats/csv.js b/lib/formats/csv.js new file mode 100644 index 00000000..4495912f --- /dev/null +++ b/lib/formats/csv.js @@ -0,0 +1,70 @@ +const fs = require('node:fs'); +const { stringify } = require('csv-stringify'); +const parse = require('csv-parse'); + +module.exports = { + label: 'CSV', + extension: '.csv', + allowedExtension: '.csv', + allowedTypes: [ 'text/csv' ], + includeAttachments: false, + importVersions: [ 'draft' ], + exportVersions: [ 'published' ], + input, + output, + formatData(docs) { + return docs; + } +}; + +async function input(filepath) { + // TODO: + // return parse({ + // columns: true, + // bom: true + // }); +}; + +async function output(apos, filepath, data) { + console.log('data'); + console.dir(data, { depth: 9 }); + + return new Promise((resolve, reject) => { + const result = []; + const stringifier = stringify({ + header: true, + columns: getColumnsNames(data) + }); + + stringifier.on('readable', function() { + let row; + while ((row = stringifier.read()) !== null) { + result.push(row); + } + }); + + stringifier.on('error', reject); + + stringifier.on('finish', function() { + console.log('result'); + console.dir(result); + resolve(result); + }); + + stringifier.pipe(fs.createWriteStream(filepath)); + + data.forEach(record => { + stringifier.write(record); + }); + + stringifier.end(); + }); +} + +function getColumnsNames(data) { + const columns = new Set(); + data.forEach(doc => { + Object.keys(doc).forEach(key => columns.add(key)); + }); + return Array.from(columns); +} diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index b59dbb7f..4ae95b21 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -4,6 +4,7 @@ const zlib = require('node:zlib'); const tar = require('tar-stream'); const stream = require('node:stream/promises'); const Promise = require('bluebird'); +const { EJSON } = require('bson'); module.exports = { label: 'gzip', @@ -13,8 +14,20 @@ module.exports = { 'application/gzip', 'application/x-gzip' ], + includeAttachments: true, + importVersions: [ 'draft', 'published' ], + exportVersions: [ 'draft', 'published' ], input, - output + output, + formatData(docs, attachments = [], attachmentsUrls = {}) { + return { + json: { + 'aposDocs.json': EJSON.stringify(docs, undefined, 2), + 'aposAttachments.json': EJSON.stringify(attachments, undefined, 2) + }, + attachments: attachmentsUrls + }; + } }; async function input(filepath) { diff --git a/lib/formats/index.js b/lib/formats/index.js new file mode 100644 index 00000000..6c1073fe --- /dev/null +++ b/lib/formats/index.js @@ -0,0 +1,7 @@ +const csv = require('./csv'); +const gzip = require('./gzip'); + +module.exports = { + csv, + gzip +}; diff --git a/lib/methods/export.js b/lib/methods/export.js index 92d70158..45e8760a 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -2,7 +2,6 @@ const fs = require('node:fs/promises'); const path = require('path'); const util = require('util'); const { uniqBy } = require('lodash'); -const { EJSON } = require('bson'); const dayjs = require('dayjs'); const MAX_RECURSION = 10; @@ -32,14 +31,14 @@ module.exports = self => { } const hasRelatedTypes = !!relatedTypes.length; - const docs = await self.getDocs(req, ids, hasRelatedTypes, manager, reporting); + const docs = await self.getDocs(req, format, ids, hasRelatedTypes, manager, reporting); const cleanDocs = self.clean(docs); if (!hasRelatedTypes) { return self.writeExportFile( req, reporting, - self.formatData(cleanDocs), + format.formatData(cleanDocs), { expiration, format @@ -56,6 +55,18 @@ module.exports = self => { ); const allCleanDocs = [ ...self.clean(relatedDocs), ...cleanDocs ]; + if (!format.includeAttachments) { + return self.writeExportFile( + req, + reporting, + format.formatData(allCleanDocs), + { + expiration, + format + } + ); + } + const attachmentsIds = uniqBy([ ...docs, ...relatedDocs ], doc => doc._id) .flatMap(doc => self.getRelatedDocsFromSchema(req, { @@ -82,7 +93,7 @@ module.exports = self => { return self.writeExportFile( req, reporting, - self.formatData(allCleanDocs, cleanAttachments, attachmentUrls), + format.formatData(allCleanDocs, cleanAttachments, attachmentUrls), { expiration, format @@ -93,35 +104,43 @@ module.exports = self => { // Get docs via their manager in order to populate them // so that we can retrieve their relationships IDs later, // and to let the manager handle permissions. - async getDocs(req, docsIds, includeRelationships, manager, reporting) { + async getDocs(req, format, docsIds, includeRelationships, manager, reporting) { if (!docsIds.length) { return []; } const { draftIds, publishedIds } = self.getAllModesIds(docsIds); const isReqDraft = req.mode === 'draft'; - const draftReq = isReqDraft ? req : req.clone({ mode: 'draft' }); - const publishedReq = isReqDraft ? req.clone({ mode: 'published' }) : req; - const draftDocs = await manager - .findForEditing(draftReq, { - _id: { - $in: draftIds - } - }) - .relationships(includeRelationships) - .toArray(); + const docs = []; - const publishedDocs = await manager - .findForEditing(publishedReq, { - _id: { - $in: publishedIds - } - }) - .relationships(includeRelationships) - .toArray(); + if (self.shouldExportDraft(format)) { + const draftReq = isReqDraft ? req : req.clone({ mode: 'draft' }); + const draftDocs = await manager + .findForEditing(draftReq, { + _id: { + $in: draftIds + } + }) + .relationships(includeRelationships) + .toArray(); + + docs.push(...draftDocs); + } + + if (self.shouldExportPublished(format)) { + const publishedReq = isReqDraft ? req.clone({ mode: 'published' }) : req; + const publishedDocs = await manager + .findForEditing(publishedReq, { + _id: { + $in: publishedIds + } + }) + .relationships(includeRelationships) + .toArray(); - const docs = [ ...draftDocs, ...publishedDocs ]; + docs.push(...publishedDocs); + } if (reporting) { const docsId = docs.map(doc => doc._id); @@ -139,6 +158,14 @@ module.exports = self => { return docs.filter(doc => self.canExport(req, doc.type)); }, + shouldExportDraft(format) { + return !format.exportVersions || !format.exportVersions.length || format.exportVersions.includes('draft'); + }, + + shouldExportPublished(format) { + return !format.exportVersions || !format.exportVersions.length || format.exportVersions.includes('published'); + }, + // Add the published version ID next to each draft ID, // so we always get both the draft and the published ID. // If somehow published IDs are sent from the frontend, @@ -208,16 +235,6 @@ module.exports = self => { return true; }, - formatData(docs, attachments = [], urls = {}) { - return { - json: { - 'aposDocs.json': EJSON.stringify(docs, undefined, 2), - 'aposAttachments.json': EJSON.stringify(attachments, undefined, 2) - }, - attachments: urls - }; - }, - getRelatedDocsFromSchema( req, { diff --git a/package.json b/package.json index 8ac20935..c7a202c8 100644 --- a/package.json +++ b/package.json @@ -33,8 +33,10 @@ "bluebird": "^3.7.2", "bson": "^6.0.0", "connect-multiparty": "^2.1.1", + "csv-parse": "^5.5.5", + "csv-stringify": "^6.4.6", "dayjs": "^1.9.8", "lodash": "^4.17.21", "tar-stream": "^3.1.6" } -} \ No newline at end of file +} From 69c786d2e95ba4f34b9c574573505c46010de927 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 3 Apr 2024 13:04:01 +0200 Subject: [PATCH 02/42] improve csv output stream --- lib/formats/csv.js | 45 +++++++++++++++---------------------------- lib/methods/export.js | 4 ++-- 2 files changed, 18 insertions(+), 31 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 4495912f..3685ae38 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -1,6 +1,6 @@ const fs = require('node:fs'); const { stringify } = require('csv-stringify'); -const parse = require('csv-parse'); +// const parse = require('csv-parse'); module.exports = { label: 'CSV', @@ -26,38 +26,25 @@ async function input(filepath) { }; async function output(apos, filepath, data) { - console.log('data'); - console.dir(data, { depth: 9 }); - - return new Promise((resolve, reject) => { - const result = []; - const stringifier = stringify({ - header: true, - columns: getColumnsNames(data) - }); - - stringifier.on('readable', function() { - let row; - while ((row = stringifier.read()) !== null) { - result.push(row); - } - }); - - stringifier.on('error', reject); + const stream = fs.createWriteStream(filepath); + const stringifier = stringify({ + header: true, + columns: getColumnsNames(data) + }); - stringifier.on('finish', function() { - console.log('result'); - console.dir(result); - resolve(result); - }); + stringifier.pipe(stream); - stringifier.pipe(fs.createWriteStream(filepath)); + // plunge each doc into the stream + data.forEach(record => { + stringifier.write(record); + }); - data.forEach(record => { - stringifier.write(record); - }); + stringifier.end(); - stringifier.end(); + return new Promise((resolve, reject) => { + stringifier.on('error', reject); + stream.on('error', reject); + stream.on('finish', resolve); }); } diff --git a/lib/methods/export.js b/lib/methods/export.js index 45e8760a..767a0f14 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -303,8 +303,8 @@ module.exports = self => { const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); try { - const { attachmentError } = await format.output(self.apos, filepath, data); - if (attachmentError) { + const result = await format.output(self.apos, filepath, data); + if (result && result.attachmentError) { await self.apos.notify(req, 'aposImportExport:exportAttachmentError', { interpolate: { format: format.label }, icon: 'alert-circle-icon', From f7366abcd9e065d4d85b56f0566748b900f5ffcb Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 3 Apr 2024 13:15:57 +0200 Subject: [PATCH 03/42] simplify conditions --- lib/methods/export.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 767a0f14..0ca99181 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -159,11 +159,11 @@ module.exports = self => { }, shouldExportDraft(format) { - return !format.exportVersions || !format.exportVersions.length || format.exportVersions.includes('draft'); + return !format.exportVersions?.length || format.exportVersions.includes('draft'); }, shouldExportPublished(format) { - return !format.exportVersions || !format.exportVersions.length || format.exportVersions.includes('published'); + return !format.exportVersions?.length || format.exportVersions.includes('published'); }, // Add the published version ID next to each draft ID, From 9d29dd970041c3dfd04f73daced6fbdce543b07a Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 3 Apr 2024 18:25:41 +0200 Subject: [PATCH 04/42] import csv WIP, an iceberg mess --- lib/formats/csv.js | 110 ++++++++---- lib/formats/gzip.js | 161 ++++++++++-------- lib/methods/import.js | 112 ++++++------ .../components/AposDuplicateImportModal.vue | 11 +- ui/apos/components/AposExportModal.vue | 4 - 5 files changed, 235 insertions(+), 163 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 3685ae38..54e6aa20 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -1,6 +1,6 @@ const fs = require('node:fs'); const { stringify } = require('csv-stringify'); -// const parse = require('csv-parse'); +const { parse } = require('csv-parse'); module.exports = { label: 'CSV', @@ -8,45 +8,93 @@ module.exports = { allowedExtension: '.csv', allowedTypes: [ 'text/csv' ], includeAttachments: false, + // TODO: remove these? because it really messes with the duplicated docs detection system importVersions: [ 'draft' ], exportVersions: [ 'published' ], - input, - output, + async input(filepath) { + return filepath; + }, + async output(apos, filepath, data) { + // console.log('🚀 ~ output ~ data:', data); + const writer = fs.createWriteStream(filepath); + const stringifier = stringify({ + header: true, + columns: getColumnsNames(data), + // quoted_string: true, + cast: { + date(value) { + return value.toISOString(); + } + } + }); + + stringifier.pipe(writer); + + // plunge each doc into the stream + data.forEach(record => { + stringifier.write(record); + }); + + stringifier.end(); + + return new Promise((resolve, reject) => { + stringifier.on('error', reject); + writer.on('error', reject); + writer.on('finish', resolve); + }); + }, formatData(docs) { return docs; - } -}; - -async function input(filepath) { - // TODO: - // return parse({ - // columns: true, - // bom: true - // }); -}; + }, + async getFilesData(exportPath, docIds) { + const reader = fs.createReadStream(exportPath); + const parser = reader + .pipe( + parse({ + columns: true, + bom: true, + cast(value, context) { + if (context.header) { + return value; + } -async function output(apos, filepath, data) { - const stream = fs.createWriteStream(filepath); - const stringifier = stringify({ - header: true, - columns: getColumnsNames(data) - }); + // console.log(context.column, value, typeof value); + // TODO: need to handle Dates, number and floats? - stringifier.pipe(stream); + try { + return JSON.parse(value); + } catch { + return value; + } + } + }) + ); - // plunge each doc into the stream - data.forEach(record => { - stringifier.write(record); - }); + const docs = []; - stringifier.end(); + parser.on('readable', function() { + let doc; + while ((doc = parser.read()) !== null) { + docs.push(doc); + } + }); - return new Promise((resolve, reject) => { - stringifier.on('error', reject); - stream.on('error', reject); - stream.on('finish', resolve); - }); -} + return new Promise((resolve, reject) => { + reader.on('error', reject); + parser.on('error', reject); + parser.on('end', () => { + // console.log('docs'); + // console.dir(docs, { depth: 9 }); + resolve({ + docs: !docIds + ? docs + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + exportPath + }); + }); + }); + } +}; function getColumnsNames(data) { const columns = new Set(); diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 4ae95b21..32138ecf 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -1,3 +1,4 @@ +const path = require('path'); const fs = require('node:fs'); const fsp = require('node:fs/promises'); const zlib = require('node:zlib'); @@ -17,8 +18,74 @@ module.exports = { includeAttachments: true, importVersions: [ 'draft', 'published' ], exportVersions: [ 'draft', 'published' ], - input, - output, + async input(filepath) { + // TODO: handle extract path when there is no actual "extraction" per se + const extractPath = filepath.replace(this.allowedExtension, ''); + + if (!fs.existsSync(extractPath)) { + await fsp.mkdir(extractPath); + } + + return new Promise((resolve, reject) => { + const input = fs.createReadStream(filepath); + const gunzip = zlib.createGunzip(); + const extract = tar.extract(); + + input.on('error', reject); + gunzip.on('error', reject); + + extract + .on('entry', (header, stream, next) => { + if (header.type === 'directory') { + fsp + .mkdir(`${extractPath}/${header.name}`) + .then(next) + .catch(reject); + } else { + stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); + stream.on('end', next); + } + }) + .on('finish', () => { + resolve(extractPath); + }) + .on('error', reject); + + input + .pipe(gunzip) + .pipe(extract); + }); + }, + async output(apos, filepath, data) { + return new Promise((resolve, reject) => { + let result; + const output = fs.createWriteStream(filepath); + const pack = tar.pack(); + const gzip = zlib.createGzip(); + + gzip.on('error', reject); + + output + .on('error', reject) + .on('finish', () => { + resolve(result); + }); + + pack + .pipe(gzip) + .pipe(output); + + for (const [ filename, content ] of Object.entries(data.json || {})) { + addTarEntry(pack, { name: filename }, content).catch(reject); + } + + compressAttachments(apos, pack, data.attachments || {}) + .then((res) => { + result = res; + pack.finalize(); + }); + }); + }, formatData(docs, attachments = [], attachmentsUrls = {}) { return { json: { @@ -27,78 +94,31 @@ module.exports = { }, attachments: attachmentsUrls }; - } -}; + }, + async getFilesData(exportPath, docIds) { + const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); + const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); -async function input(filepath) { - const extractPath = filepath.replace(this.allowedExtension, ''); - - if (!fs.existsSync(extractPath)) { - await fsp.mkdir(extractPath); - } - - return new Promise((resolve, reject) => { - const input = fs.createReadStream(filepath); - const gunzip = zlib.createGunzip(); - const extract = tar.extract(); - - input.on('error', reject); - gunzip.on('error', reject); - - extract - .on('entry', (header, stream, next) => { - if (header.type === 'directory') { - fsp - .mkdir(`${extractPath}/${header.name}`) - .then(next) - .catch(reject); - } else { - stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); - stream.on('end', next); + return { + docs: !docIds + ? EJSON.parse(docs) + : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join( + exportPath, + 'attachments', + `${attachment._id}-${attachment.name}.${attachment.extension}` + ) } - }) - .on('finish', () => { - resolve(extractPath); - }) - .on('error', reject); - - input - .pipe(gunzip) - .pipe(extract); - }); + })), + exportPath + }; + } }; -async function output(apos, filepath, data) { - return new Promise((resolve, reject) => { - let result; - const output = fs.createWriteStream(filepath); - const pack = tar.pack(); - const gzip = zlib.createGzip(); - - gzip.on('error', reject); - - output - .on('error', reject) - .on('finish', () => { - resolve(result); - }); - - pack - .pipe(gzip) - .pipe(output); - - for (const [ filename, content ] of Object.entries(data.json || {})) { - addTarEntry(pack, { name: filename }, content).catch(reject); - } - - compressAttachments(apos, pack, data.attachments || {}) - .then((res) => { - result = res; - pack.finalize(); - }); - }); -} - function addTarEntry(pack, options, data = null) { return new Promise((resolve, reject) => { pack.entry(options, data, (err) => { @@ -112,7 +132,6 @@ function addTarEntry(pack, options, data = null) { } async function compressAttachments(apos, pack, attachments = {}) { - const copyOut = Promise.promisify(apos.attachment.uploadfs.copyOut); await addTarEntry(pack, { diff --git a/lib/methods/import.js b/lib/methods/import.js index 2e678841..eca405f5 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -1,7 +1,5 @@ -const path = require('path'); const fsp = require('node:fs/promises'); const { cloneDeep } = require('lodash'); -const { EJSON } = require('bson'); module.exports = self => { return { @@ -10,6 +8,20 @@ module.exports = self => { throw self.apos.error('forbidden'); } + const { file } = req.files || {}; + + if (!file) { + throw self.apos.error('invalid'); + } + + const format = Object + .values(self.formats) + .find((format) => format.allowedTypes.includes(file.type)); + + if (!format) { + throw self.apos.error('invalid'); + } + const overrideLocale = self.apos.launder.boolean(req.body.overrideLocale); let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); @@ -17,11 +29,12 @@ module.exports = self => { let attachmentsInfo; if (exportPath) { - const filesData = await self.getFilesData(exportPath); + const filesData = await format.getFilesData(exportPath); ({ docs, attachmentsInfo } = filesData); } else { - const exportFile = await self.readExportFile(req); + const exportFile = await self.readExportFile(req, format, file); + console.log('🚀 ~ import ~ exportFile:', exportFile); ({ docs, attachmentsInfo } = exportFile); exportPath = exportFile.exportPath; @@ -65,21 +78,30 @@ module.exports = self => { self.rewriteDocsWithCurrentLocale(req, docs); } - const total = docs.length + attachmentsInfo.length; + const total = format.includeAttachments && attachmentsInfo?.length + ? docs.length + attachmentsInfo.length + : docs.length; + const { reporting, jobId, notificationId } = await self.instantiateJob(req, total); + // console.dir(docs, { depth: 9 }); + const { duplicatedDocs, duplicatedIds, failedIds } = await self.insertDocs(req, docs, reporting); - const importedAttachments = await self.insertAttachments(req, { - attachmentsInfo, - reporting, - duplicatedIds, - docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) - }); + let importedAttachments = []; + + if (format.includeAttachments && attachmentsInfo?.length) { + importedAttachments = await self.insertAttachments(req, { + attachmentsInfo, + reporting, + duplicatedIds, + docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) + }); + } if (!duplicatedDocs.length) { await reporting.end(); @@ -126,7 +148,8 @@ module.exports = self => { type: moduleName, exportPathId: await self.getExportPathId(exportPath), jobId, - notificationId + notificationId, + formatLabel: format.label }; // we only care about the event here, @@ -175,27 +198,14 @@ module.exports = self => { }; }, - async readExportFile(req) { - const { file } = req.files || {}; - - if (!file) { - throw self.apos.error('invalid'); - } - - const format = Object - .values(self.formats) - .find((format) => format.allowedTypes.includes(file.type)); - - if (!format) { - throw self.apos.error('invalid'); - } - + async readExportFile(req, format, file) { try { const exportPath = await format.input(file.path); + console.log('🚀 ~ readExportFile ~ exportPath:', exportPath); await self.setExportPathId(exportPath); return { - ...await self.getFilesData(exportPath), + ...await format.getFilesData(exportPath), filePath: file.path }; } catch (error) { @@ -209,29 +219,6 @@ module.exports = self => { } }, - async getFilesData(exportPath, docIds) { - const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); - const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); - - return { - docs: !docIds - ? EJSON.parse(docs) - : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - attachment, - file: { - name: `${attachment.name}.${attachment.extension}`, - path: path.join( - exportPath, - 'attachments', - `${attachment._id}-${attachment.name}.${attachment.extension}` - ) - } - })), - exportPath - }; - }, - getFirstDifferentLocale(req, docs) { const doc = docs .find(doc => self.isLocaleDifferent(req, doc)); @@ -260,6 +247,7 @@ module.exports = self => { return locale; }, + // TODO: duplicatedDocs are not detected async insertDocs(req, docs, reporting) { const duplicatedDocs = []; const duplicatedIds = []; @@ -394,6 +382,14 @@ module.exports = self => { return manager[method](_req, doc, { setModified: false }); }, + shouldImportDraft(format) { + return !format.importVersions?.length || format.importVersions.includes('draft'); + }, + + shouldImportPublished(format) { + return !format.importVersions?.length || format.importVersions.includes('published'); + }, + async insertOrUpdateAttachment(req, { attachmentInfo: { attachment, file }, duplicatedIds, docIds }) { @@ -445,6 +441,7 @@ module.exports = self => { await fsp.unlink(path); } } catch (err) { + console.trace(); self.apos.util.error( `Error while trying to remove the file or folder: ${path}. You might want to remove it yourself.` ); @@ -457,12 +454,23 @@ module.exports = self => { const docIds = self.apos.launder.strings(req.body.docIds); const jobId = self.apos.launder.string(req.body.jobId); const importedAttachments = self.apos.launder.strings(req.body.importedAttachments); + const formatLabel = self.apos.launder.string(req.body.formatLabel); const failedIds = []; + // console.log('🚀 ~ overrideDuplicates ~ formatLabel:', formatLabel); const jobManager = self.apos.modules['@apostrophecms/job']; const job = await jobManager.db.findOne({ _id: jobId }); - const { docs, attachmentsInfo } = await self.getFilesData(exportPath, docIds); + const format = Object + .values(self.formats) + .find(format => format.label === formatLabel); + + if (!format) { + jobManager.failure(job); + throw self.apos.error(`invalid format "${formatLabel}"`); + } + + const { docs, attachmentsInfo } = await format.getFilesData(exportPath, docIds); const differentDocsLocale = self.getFirstDifferentLocale(req, docs); const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1; diff --git a/ui/apos/components/AposDuplicateImportModal.vue b/ui/apos/components/AposDuplicateImportModal.vue index e15bbd40..9860a61d 100644 --- a/ui/apos/components/AposDuplicateImportModal.vue +++ b/ui/apos/components/AposDuplicateImportModal.vue @@ -127,6 +127,10 @@ export default { type: Array, required: true }, + formatLabel: { + type: String, + required: true + }, jobId: { type: String, required: true @@ -216,7 +220,8 @@ export default { importedAttachments: this.importedAttachments, exportPathId: this.exportPathId, jobId: this.jobId, - overrideLocale: this.overrideLocale + overrideLocale: this.overrideLocale, + formatLabel: this.formatLabel } }).catch(() => { apos.notify('aposImportExport:exportFailed', { @@ -293,10 +298,6 @@ export default { display: flex; } -:deep(.apos-input--select) { - text-transform: capitalize; -} - .apos-import-duplicate__heading { @include type-title; line-height: var(--a-line-tall); diff --git a/ui/apos/components/AposExportModal.vue b/ui/apos/components/AposExportModal.vue index 466ccaba..325af13e 100644 --- a/ui/apos/components/AposExportModal.vue +++ b/ui/apos/components/AposExportModal.vue @@ -330,10 +330,6 @@ export default { display: flex; } -:deep(.apos-input--select) { - text-transform: capitalize; -} - .apos-export__heading { @include type-title; line-height: var(--a-line-tall); From 320d4e08db45c0171584fc35fb0ae0ace8027480 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Thu, 4 Apr 2024 11:52:51 +0200 Subject: [PATCH 05/42] wip --- lib/formats/csv.js | 9 +++----- lib/formats/gzip.js | 7 +++--- lib/methods/import.js | 54 +++++++++++++++---------------------------- 3 files changed, 25 insertions(+), 45 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 54e6aa20..a999e2d4 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -9,8 +9,8 @@ module.exports = { allowedTypes: [ 'text/csv' ], includeAttachments: false, // TODO: remove these? because it really messes with the duplicated docs detection system - importVersions: [ 'draft' ], - exportVersions: [ 'published' ], + // importVersions: [ 'draft' ], + // exportVersions: [ 'published' ], async input(filepath) { return filepath; }, @@ -83,13 +83,10 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { - // console.log('docs'); - // console.dir(docs, { depth: 9 }); resolve({ docs: !docIds ? docs - : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), - exportPath + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)) }); }); }); diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 32138ecf..d515fc63 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -16,8 +16,8 @@ module.exports = { 'application/x-gzip' ], includeAttachments: true, - importVersions: [ 'draft', 'published' ], - exportVersions: [ 'draft', 'published' ], + // importVersions: [ 'draft', 'published' ], + // exportVersions: [ 'draft', 'published' ], async input(filepath) { // TODO: handle extract path when there is no actual "extraction" per se const extractPath = filepath.replace(this.allowedExtension, ''); @@ -113,8 +113,7 @@ module.exports = { `${attachment._id}-${attachment.name}.${attachment.extension}` ) } - })), - exportPath + })) }; } }; diff --git a/lib/methods/import.js b/lib/methods/import.js index eca405f5..f6cb2f35 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -26,20 +26,27 @@ module.exports = self => { let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); let docs; - let attachmentsInfo; + let attachmentsInfo = []; - if (exportPath) { + try { + if (!exportPath) { + exportPath = await format.input(file.path); + await self.setExportPathId(exportPath); + } const filesData = await format.getFilesData(exportPath); - ({ docs, attachmentsInfo } = filesData); - } else { - const exportFile = await self.readExportFile(req, format, file); - console.log('🚀 ~ import ~ exportFile:', exportFile); - - ({ docs, attachmentsInfo } = exportFile); - exportPath = exportFile.exportPath; + docs = filesData.docs; + attachmentsInfo = filesData.attachmentsInfo; - await self.cleanFile(exportFile.filePath); + await self.cleanFile(file.path); + } catch (error) { + await self.apos.notify(req, 'aposImportExport:importFileError', { + interpolate: { format: format.label }, + dismiss: true, + icon: 'alert-circle-icon', + type: 'danger' + }); + throw self.apos.error(error.message); } const differentDocsLocale = self.getFirstDifferentLocale(req, docs); @@ -78,7 +85,7 @@ module.exports = self => { self.rewriteDocsWithCurrentLocale(req, docs); } - const total = format.includeAttachments && attachmentsInfo?.length + const total = format.includeAttachments ? docs.length + attachmentsInfo.length : docs.length; @@ -86,15 +93,13 @@ module.exports = self => { reporting, jobId, notificationId } = await self.instantiateJob(req, total); - // console.dir(docs, { depth: 9 }); - const { duplicatedDocs, duplicatedIds, failedIds } = await self.insertDocs(req, docs, reporting); let importedAttachments = []; - if (format.includeAttachments && attachmentsInfo?.length) { + if (format.includeAttachments) { importedAttachments = await self.insertAttachments(req, { attachmentsInfo, reporting, @@ -198,27 +203,6 @@ module.exports = self => { }; }, - async readExportFile(req, format, file) { - try { - const exportPath = await format.input(file.path); - console.log('🚀 ~ readExportFile ~ exportPath:', exportPath); - await self.setExportPathId(exportPath); - - return { - ...await format.getFilesData(exportPath), - filePath: file.path - }; - } catch (error) { - await self.apos.notify(req, 'aposImportExport:importFileError', { - interpolate: { format: format.label }, - dismiss: true, - icon: 'alert-circle-icon', - type: 'danger' - }); - throw self.apos.error(error.message); - } - }, - getFirstDifferentLocale(req, docs) { const doc = docs .find(doc => self.isLocaleDifferent(req, doc)); From 5fbdf54293619e8e67ae9c110aae9800ba02c3c4 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Fri, 5 Apr 2024 14:45:59 +0200 Subject: [PATCH 06/42] move canImport to import.js and canImportOrExport to index.js --- lib/methods/export.js | 24 ------------------------ lib/methods/import.js | 4 ++++ lib/methods/index.js | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 24 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index 0ca99181..2f457117 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -207,34 +207,10 @@ module.exports = self => { .toArray(); }, - canImport(req, docType) { - return self.canImportOrExport(req, docType, 'import'); - }, - canExport(req, docType) { return self.canImportOrExport(req, docType, 'export'); }, - // Filter our docs that have their module with the import or export option set to false - // and docs that have "admin only" permissions when the user is not an admin. - // If a user does not have at lease the permission to view the draft, he won't - // be able to import or export it. - canImportOrExport(req, docType, action) { - const docModule = self.apos.modules[docType]; - - if (!docModule) { - return false; - } - if (docModule.options.importExport?.[action] === false) { - return false; - } - if (!self.apos.permission.can(req, 'view', docType)) { - return false; - } - - return true; - }, - getRelatedDocsFromSchema( req, { diff --git a/lib/methods/import.js b/lib/methods/import.js index f6cb2f35..0c8e6f64 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -374,6 +374,10 @@ module.exports = self => { return !format.importVersions?.length || format.importVersions.includes('published'); }, + canImport(req, docType) { + return self.canImportOrExport(req, docType, 'import'); + }, + async insertOrUpdateAttachment(req, { attachmentInfo: { attachment, file }, duplicatedIds, docIds }) { diff --git a/lib/methods/index.js b/lib/methods/index.js index 767eba74..f8d2e159 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -15,6 +15,25 @@ module.exports = self => { })) }; }, + // Filter our docs that have their module with the import or export option set to false + // and docs that have "admin only" permissions when the user is not an admin. + // If a user does not have at lease the permission to view the draft, he won't + // be able to import or export it. + canImportOrExport(req, docType, action) { + const docModule = self.apos.modules[docType]; + + if (!docModule) { + return false; + } + if (docModule.options.importExport?.[action] === false) { + return false; + } + if (!self.apos.permission.can(req, 'view', docType)) { + return false; + } + + return true; + }, ...importMethods(self), ...exportMethods(self) }; From 04a534339c9d069b41c7aea28ae37e483388c1a0 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Fri, 5 Apr 2024 15:00:23 +0200 Subject: [PATCH 07/42] export: separate concerns between apos related stuff and formats --- lib/formats/csv.js | 69 +++++++-------- lib/formats/gzip.js | 197 +++++++++++++++++++----------------------- lib/methods/export.js | 151 +++++++++++++++++++------------- 3 files changed, 211 insertions(+), 206 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index a999e2d4..e6e6f948 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -11,42 +11,7 @@ module.exports = { // TODO: remove these? because it really messes with the duplicated docs detection system // importVersions: [ 'draft' ], // exportVersions: [ 'published' ], - async input(filepath) { - return filepath; - }, - async output(apos, filepath, data) { - // console.log('🚀 ~ output ~ data:', data); - const writer = fs.createWriteStream(filepath); - const stringifier = stringify({ - header: true, - columns: getColumnsNames(data), - // quoted_string: true, - cast: { - date(value) { - return value.toISOString(); - } - } - }); - - stringifier.pipe(writer); - - // plunge each doc into the stream - data.forEach(record => { - stringifier.write(record); - }); - - stringifier.end(); - - return new Promise((resolve, reject) => { - stringifier.on('error', reject); - writer.on('error', reject); - writer.on('finish', resolve); - }); - }, - formatData(docs) { - return docs; - }, - async getFilesData(exportPath, docIds) { + async import(exportPath) { const reader = fs.createReadStream(exportPath); const parser = reader .pipe( @@ -90,12 +55,40 @@ module.exports = { }); }); }); + }, + async export(filepath, { docs }) { + const writer = fs.createWriteStream(filepath); + const stringifier = stringify({ + header: true, + columns: getColumnsNames(docs), + // quoted_string: true, + cast: { + date(value) { + return value.toISOString(); + } + } + }); + + stringifier.pipe(writer); + + // plunge each doc into the stream + docs.forEach(record => { + stringifier.write(record); + }); + + stringifier.end(); + + return new Promise((resolve, reject) => { + stringifier.on('error', reject); + writer.on('error', reject); + writer.on('finish', resolve); + }); } }; -function getColumnsNames(data) { +function getColumnsNames(docs) { const columns = new Set(); - data.forEach(doc => { + docs.forEach(doc => { Object.keys(doc).forEach(key => columns.add(key)); }); return Array.from(columns); diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index d515fc63..d1dcb937 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -1,10 +1,9 @@ -const path = require('path'); +const path = require('node:path'); const fs = require('node:fs'); const fsp = require('node:fs/promises'); +const stream = require('node:stream/promises'); const zlib = require('node:zlib'); const tar = require('tar-stream'); -const stream = require('node:stream/promises'); -const Promise = require('bluebird'); const { EJSON } = require('bson'); module.exports = { @@ -18,84 +17,108 @@ module.exports = { includeAttachments: true, // importVersions: [ 'draft', 'published' ], // exportVersions: [ 'draft', 'published' ], - async input(filepath) { - // TODO: handle extract path when there is no actual "extraction" per se + async import(filepath) { const extractPath = filepath.replace(this.allowedExtension, ''); if (!fs.existsSync(extractPath)) { await fsp.mkdir(extractPath); } - return new Promise((resolve, reject) => { - const input = fs.createReadStream(filepath); - const gunzip = zlib.createGunzip(); - const extract = tar.extract(); + const readStream = fs.createReadStream(filepath); + const gunzip = zlib.createGunzip(); + const extract = tar.extract(); - input.on('error', reject); - gunzip.on('error', reject); + readStream + .pipe(gunzip) + .pipe(extract); - extract - .on('entry', (header, stream, next) => { - if (header.type === 'directory') { - fsp - .mkdir(`${extractPath}/${header.name}`) - .then(next) - .catch(reject); - } else { - stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); - stream.on('end', next); - } - }) - .on('finish', () => { - resolve(extractPath); - }) - .on('error', reject); + return new Promise((resolve, reject) => { + readStream.on('error', reject); + gunzip.on('error', reject); + extract.on('error', reject); + + extract.on('entry', (header, stream, next) => { + if (header.type === 'directory') { + fsp + .mkdir(`${extractPath}/${header.name}`) + .then(next) + .catch(reject); + } else { + stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); + stream.on('end', next); + } + }); - input - .pipe(gunzip) - .pipe(extract); + extract.on('finish', () => { + resolve(extractPath); + }); }); }, - async output(apos, filepath, data) { - return new Promise((resolve, reject) => { - let result; - const output = fs.createWriteStream(filepath); - const pack = tar.pack(); - const gzip = zlib.createGzip(); + async export( + filepath, + { + docs, + attachments = [], + attachmentUrls = {} + }, + processAttachments + ) { + const data = { + json: { + 'aposDocs.json': EJSON.stringify(docs, undefined, 2), + 'aposAttachments.json': EJSON.stringify(attachments, undefined, 2) + }, + attachments: attachmentUrls + }; - gzip.on('error', reject); + const writeStream = fs.createWriteStream(filepath); + const pack = tar.pack(); + const gzip = zlib.createGzip(); - output - .on('error', reject) - .on('finish', () => { - resolve(result); - }); + pack + .pipe(gzip) + .pipe(writeStream); - pack - .pipe(gzip) - .pipe(output); + let result; + + return new Promise((resolve, reject) => { + writeStream.on('error', reject); + gzip.on('error', reject); + pack.on('error', reject); + + writeStream.on('finish', () => { + resolve(result); + }); for (const [ filename, content ] of Object.entries(data.json || {})) { addTarEntry(pack, { name: filename }, content).catch(reject); } - compressAttachments(apos, pack, data.attachments || {}) - .then((res) => { - result = res; - pack.finalize(); - }); + addTarEntry(pack, { + name: 'attachments/', + type: 'directory' + }) + .then(() => { + console.log('attachments'); + processAttachments(data.attachments, async (temp, name, size) => { + console.log('callback', { temp, name, size }); + const readStream = fs.createReadStream(temp); + const entryStream = pack.entry({ + name: `attachments/${name}`, + size + }); + + await stream.pipeline([ readStream, entryStream ]); + }) + .then((res) => { + result = res; + pack.finalize(); + }); + }) + .catch(reject); }); }, - formatData(docs, attachments = [], attachmentsUrls = {}) { - return { - json: { - 'aposDocs.json': EJSON.stringify(docs, undefined, 2), - 'aposAttachments.json': EJSON.stringify(attachments, undefined, 2) - }, - attachments: attachmentsUrls - }; - }, - async getFilesData(exportPath, docIds) { + async read(exportPath, docIds) { const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); @@ -120,59 +143,13 @@ module.exports = { function addTarEntry(pack, options, data = null) { return new Promise((resolve, reject) => { - pack.entry(options, data, (err) => { - if (err) { - reject(err); + pack.entry(options, data, error => { + if (error) { + reject(error); + return; } resolve(); }); }); } - -async function compressAttachments(apos, pack, attachments = {}) { - const copyOut = Promise.promisify(apos.attachment.uploadfs.copyOut); - - await addTarEntry(pack, { - name: 'attachments/', - type: 'directory' - }); - - let attachmentError = false; - - await Promise.map(Object.entries(attachments), processOneAttachment, { - concurrency: 5 - }); - - return { attachmentError }; - - async function processOneAttachment([ name, url ]) { - const temp = apos.attachment.uploadfs.getTempPath() + '/' + apos.util.generateId(); - try { - await copyOut(url, temp); - const size = fs.statSync(temp).size; - // Looking at the source code, the tar-stream module - // probably doesn't protect against two input streams - // pushing mishmashed bytes into the tarball at the - // same time, so stream in just one at a time. We still - // get good concurrency on copyOut which is much slower - // than this operation - await apos.lock.withLock('import-export-copy-out', async () => { - const fileStream = fs.createReadStream(temp); - // Use the stream-based API - const entryStream = pack.entry({ - name: `attachments/${name}`, - size - }); - await stream.pipeline([ fileStream, entryStream ]); - }); - } catch (e) { - attachmentError = true; - apos.util.error(e); - } finally { - if (fs.existsSync(temp)) { - fs.unlinkSync(temp); - } - } - } -} diff --git a/lib/methods/export.js b/lib/methods/export.js index 2f457117..72a8a382 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -1,8 +1,9 @@ -const fs = require('node:fs/promises'); -const path = require('path'); -const util = require('util'); -const { uniqBy } = require('lodash'); +const path = require('node:path'); +const util = require('node:util'); +const fsp = require('node:fs/promises'); +const Promise = require('bluebird'); const dayjs = require('dayjs'); +const { uniqBy } = require('lodash'); const MAX_RECURSION = 10; @@ -31,18 +32,16 @@ module.exports = self => { } const hasRelatedTypes = !!relatedTypes.length; - const docs = await self.getDocs(req, format, ids, hasRelatedTypes, manager, reporting); + const docs = await self.getDocs(req, ids, hasRelatedTypes, manager, reporting); const cleanDocs = self.clean(docs); if (!hasRelatedTypes) { - return self.writeExportFile( + return self.exportFile( req, reporting, - format.formatData(cleanDocs), - { - expiration, - format - } + format, + { docs: cleanDocs }, + expiration ); } @@ -56,14 +55,12 @@ module.exports = self => { const allCleanDocs = [ ...self.clean(relatedDocs), ...cleanDocs ]; if (!format.includeAttachments) { - return self.writeExportFile( + return self.exportFile( req, reporting, - format.formatData(allCleanDocs), - { - expiration, - format - } + format, + { docs: allCleanDocs }, + expiration ); } @@ -77,7 +74,7 @@ module.exports = self => { ) .map(attachment => attachment._id); - const attachments = await self.getAttachments(req, attachmentsIds); + const attachments = await self.getAttachments(attachmentsIds); const cleanAttachments = self.clean(attachments); const attachmentUrls = Object.fromEntries( @@ -90,21 +87,23 @@ module.exports = self => { }) ); - return self.writeExportFile( + return self.exportFile( req, reporting, - format.formatData(allCleanDocs, cleanAttachments, attachmentUrls), + format, { - expiration, - format - } + docs: allCleanDocs, + attachments: cleanAttachments, + attachmentUrls + }, + expiration ); }, // Get docs via their manager in order to populate them // so that we can retrieve their relationships IDs later, // and to let the manager handle permissions. - async getDocs(req, format, docsIds, includeRelationships, manager, reporting) { + async getDocs(req, docsIds, includeRelationships, manager, reporting) { if (!docsIds.length) { return []; } @@ -114,33 +113,29 @@ module.exports = self => { const docs = []; - if (self.shouldExportDraft(format)) { - const draftReq = isReqDraft ? req : req.clone({ mode: 'draft' }); - const draftDocs = await manager - .findForEditing(draftReq, { - _id: { - $in: draftIds - } - }) - .relationships(includeRelationships) - .toArray(); + const draftReq = isReqDraft ? req : req.clone({ mode: 'draft' }); + const draftDocs = await manager + .findForEditing(draftReq, { + _id: { + $in: draftIds + } + }) + .relationships(includeRelationships) + .toArray(); - docs.push(...draftDocs); - } + docs.push(...draftDocs); - if (self.shouldExportPublished(format)) { - const publishedReq = isReqDraft ? req.clone({ mode: 'published' }) : req; - const publishedDocs = await manager - .findForEditing(publishedReq, { - _id: { - $in: publishedIds - } - }) - .relationships(includeRelationships) - .toArray(); + const publishedReq = isReqDraft ? req.clone({ mode: 'published' }) : req; + const publishedDocs = await manager + .findForEditing(publishedReq, { + _id: { + $in: publishedIds + } + }) + .relationships(includeRelationships) + .toArray(); - docs.push(...publishedDocs); - } + docs.push(...publishedDocs); if (reporting) { const docsId = docs.map(doc => doc._id); @@ -193,7 +188,7 @@ module.exports = self => { .map(doc => self.apos.util.clonePermanent(doc)); }, - getAttachments(req, ids) { + getAttachments(ids) { if (!ids.length) { return []; } @@ -273,14 +268,15 @@ module.exports = self => { }); }, - async writeExportFile(req, reporting, data, { format, expiration }) { + async exportFile(req, reporting, format, data, expiration) { const date = dayjs().format('YYYYMMDDHHmmss'); const filename = `${self.apos.shortName}-${req.body.type.toLowerCase()}-export-${date}${format.extension}`; const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); try { - const result = await format.output(self.apos, filepath, data); - if (result && result.attachmentError) { + const result = await format.export(filepath, data, self.processAttachments); + + if (format.includeAttachments && result?.attachmentError) { await self.apos.notify(req, 'aposImportExport:exportAttachmentError', { interpolate: { format: format.label }, icon: 'alert-circle-icon', @@ -304,7 +300,7 @@ module.exports = self => { try { await copyIn(filepath, downloadPath); } catch (error) { - await self.removeExportFile(filepath); + await self.remove(filepath); throw error; } @@ -336,22 +332,61 @@ module.exports = self => { }); } - await self.removeExportFile(filepath); - self.removeExportFileFromUploadFs(downloadPath, expiration); + await self.remove(filepath); + self.removeFromUploadFs(downloadPath, expiration); return { url: downloadUrl }; }, - async removeExportFile(filepath) { + async processAttachments(attachments, addAttachment) { + let attachmentError = false; + + console.log('🚀 ~ processAttachments ~ attachments:', attachments); + const copyOut = Promise.promisify(self.apos.attachment.uploadfs.copyOut); + + await Promise.map(Object.entries(attachments), processAttachment, { + concurrency: 5 + }); + + return { attachmentError }; + + async function processAttachment([ name, url ]) { + console.log('🚀 ~ processAttachment ~ [ name, url ]:', [ name, url ]); + const temp = self.apos.attachment.uploadfs.getTempPath() + '/' + self.apos.util.generateId(); + try { + await copyOut(url, temp); + const { size } = await fsp.stat(temp); + console.log('🚀 ~ processAttachment ~ size:', size); + // Looking at the source code, the tar-stream module + // probably doesn't protect against two input streams + // pushing mishmashed bytes into the tarball at the + // same time, so stream in just one at a time. We still + // get good concurrency on copyOut which is much slower + // than this operation + await self.apos.lock.withLock('import-export-copy-out', async () => { + // Add attachment into the specific format + console.log('addAttachment', { temp, name, size }); + await addAttachment(temp, name, size); + }); + } catch (e) { + attachmentError = true; + self.apos.util.error(e); + } finally { + await self.remove(temp); + } + } + }, + + async remove(filepath) { try { - await fs.unlink(filepath); + await fsp.unlink(filepath); } catch (error) { self.apos.util.error(error); } }, // Report is available for 10 minutes by default - removeExportFileFromUploadFs(downloadPath, expiration) { + removeFromUploadFs(downloadPath, expiration) { setTimeout(() => { self.apos.attachment.uploadfs.remove(downloadPath, error => { if (error) { From 8c35515e688ace561abaed047f25998c9fdc77aa Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Fri, 5 Apr 2024 15:17:32 +0200 Subject: [PATCH 08/42] comment --- lib/methods/import.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index 0c8e6f64..e84bba4d 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -231,7 +231,7 @@ module.exports = self => { return locale; }, - // TODO: duplicatedDocs are not detected + // TODO: duplicatedDocs are not detected when importing as published only async insertDocs(req, docs, reporting) { const duplicatedDocs = []; const duplicatedIds = []; From 0f015b38f3246b23899dd70ee6303c4975ef7281 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Fri, 5 Apr 2024 18:58:33 +0200 Subject: [PATCH 09/42] import: separate concerns between apos related stuff and formats --- lib/formats/csv.js | 16 +++-- lib/formats/gzip.js | 145 +++++++++++++++++++++++++++--------------- lib/methods/import.js | 121 ++++++++++++++++++----------------- 3 files changed, 168 insertions(+), 114 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index e6e6f948..1bf19e43 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -11,7 +11,7 @@ module.exports = { // TODO: remove these? because it really messes with the duplicated docs detection system // importVersions: [ 'draft' ], // exportVersions: [ 'published' ], - async import(exportPath) { + async import(exportPath, { docIds } = {}) { const reader = fs.createReadStream(exportPath); const parser = reader .pipe( @@ -48,16 +48,24 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { + console.log('IMPORT RETURNS:'); + console.dir({ + docs: !docIds + ? docs + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + exportPath + }, { depth: 9 }); resolve({ docs: !docIds ? docs - : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)) + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + exportPath }); }); }); }, - async export(filepath, { docs }) { - const writer = fs.createWriteStream(filepath); + async export(exportPath, { docs }) { + const writer = fs.createWriteStream(exportPath); const stringifier = stringify({ header: true, columns: getColumnsNames(docs), diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index d1dcb937..260ad445 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -17,42 +17,59 @@ module.exports = { includeAttachments: true, // importVersions: [ 'draft', 'published' ], // exportVersions: [ 'draft', 'published' ], - async import(filepath) { - const extractPath = filepath.replace(this.allowedExtension, ''); + async import(exportPath, { docIds } = {}) { + console.log('🚀 ~ import ~ exportPath:', exportPath); + console.log('🚀 ~ import ~ docIds:', docIds); + console.log('this.allowedExtension', this.allowedExtension); + // If the given path is actually the archive, we first need to extract it. + // Then we no longer need the archive file, so we remove it. + if (exportPath.endsWith(this.allowedExtension)) { + const filepath = exportPath; + exportPath = exportPath.replace(this.allowedExtension, ''); + await extract(filepath, exportPath); + await remove(filepath); - if (!fs.existsSync(extractPath)) { - await fsp.mkdir(extractPath); } + console.log('🚀 ~ import ~ exportPath:', exportPath); - const readStream = fs.createReadStream(filepath); - const gunzip = zlib.createGunzip(); - const extract = tar.extract(); - - readStream - .pipe(gunzip) - .pipe(extract); - - return new Promise((resolve, reject) => { - readStream.on('error', reject); - gunzip.on('error', reject); - extract.on('error', reject); - - extract.on('entry', (header, stream, next) => { - if (header.type === 'directory') { - fsp - .mkdir(`${extractPath}/${header.name}`) - .then(next) - .catch(reject); - } else { - stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); - stream.on('end', next); + const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); + const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); + console.log('IMPORT RETURNS:'); + console.dir({ + docs: !docIds + ? EJSON.parse(docs) + : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join( + exportPath, + 'attachments', + `${attachment._id}-${attachment.name}.${attachment.extension}` + ) } - }); + })), + exportPath + }, { depth: 9 }); - extract.on('finish', () => { - resolve(extractPath); - }); - }); + return { + docs: !docIds + ? EJSON.parse(docs) + : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join( + exportPath, + 'attachments', + `${attachment._id}-${attachment.name}.${attachment.extension}` + ) + } + })), + exportPath + }; }, async export( filepath, @@ -117,30 +134,54 @@ module.exports = { }) .catch(reject); }); - }, - async read(exportPath, docIds) { - const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); - const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); - - return { - docs: !docIds - ? EJSON.parse(docs) - : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - attachment, - file: { - name: `${attachment.name}.${attachment.extension}`, - path: path.join( - exportPath, - 'attachments', - `${attachment._id}-${attachment.name}.${attachment.extension}` - ) - } - })) - }; } }; +async function extract(filepath, exportPath) { + console.log('🚀 ~ extract ~ filepath:', filepath); + console.log('🚀 ~ extract ~ exportPath:', exportPath); + + if (!fs.existsSync(exportPath)) { + await fsp.mkdir(exportPath); + } + + const readStream = fs.createReadStream(filepath); + const gunzip = zlib.createGunzip(); + const extract = tar.extract(); + + readStream + .pipe(gunzip) + .pipe(extract); + + return new Promise((resolve, reject) => { + readStream.on('error', reject); + gunzip.on('error', reject); + extract.on('error', reject); + + extract.on('entry', (header, stream, next) => { + if (header.type === 'directory') { + fsp + .mkdir(path.join(exportPath, header.name)) + .then(next) + .catch(reject); + } else { + stream.pipe(fs.WriteStream(path.join(exportPath, header.name))); + stream.on('end', next); + } + }); + extract.on('finish', resolve); + }); +} + +async function remove(filepath) { + console.log('🚀 ~ remove ~ filepath:', filepath); + try { + await fsp.unlink(filepath); + } catch (error) { + self.apos.util.error(error); + } +} + function addTarEntry(pack, options, data = null) { return new Promise((resolve, reject) => { pack.entry(options, data, error => { diff --git a/lib/methods/import.js b/lib/methods/import.js index e84bba4d..ec648ecc 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -26,19 +26,23 @@ module.exports = self => { let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); let docs; - let attachmentsInfo = []; + let attachmentsInfo; try { - if (!exportPath) { - exportPath = await format.input(file.path); + if (exportPath) { + ({ + docs = [], + attachmentsInfo = [] + } = await format.import(exportPath)); + } else { + ({ + docs = [], + attachmentsInfo = [], + exportPath + } = await format.import(file.path)); + await self.setExportPathId(exportPath); } - const filesData = await format.getFilesData(exportPath); - - docs = filesData.docs; - attachmentsInfo = filesData.attachmentsInfo; - - await self.cleanFile(file.path); } catch (error) { await self.apos.notify(req, 'aposImportExport:importFileError', { interpolate: { format: format.label }, @@ -85,9 +89,7 @@ module.exports = self => { self.rewriteDocsWithCurrentLocale(req, docs); } - const total = format.includeAttachments - ? docs.length + attachmentsInfo.length - : docs.length; + const total = docs.length + attachmentsInfo.length; const { reporting, jobId, notificationId @@ -97,16 +99,12 @@ module.exports = self => { duplicatedDocs, duplicatedIds, failedIds } = await self.insertDocs(req, docs, reporting); - let importedAttachments = []; - - if (format.includeAttachments) { - importedAttachments = await self.insertAttachments(req, { - attachmentsInfo, - reporting, - duplicatedIds, - docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) - }); - } + const importedAttachments = await self.insertAttachments(req, { + attachmentsInfo, + reporting, + duplicatedIds, + docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) + }); if (!duplicatedDocs.length) { await reporting.end(); @@ -131,7 +129,7 @@ module.exports = self => { self.apos.util.error(error); }); - await self.cleanFile(exportPath); + await self.remove(exportPath); return; } @@ -417,25 +415,6 @@ module.exports = self => { } }, - async cleanFile(path) { - try { - const stat = await fsp.lstat(path); - if (stat.isDirectory()) { - await fsp.rm(path, { - recursive: true, - force: true - }); - } else { - await fsp.unlink(path); - } - } catch (err) { - console.trace(); - self.apos.util.error( - `Error while trying to remove the file or folder: ${path}. You might want to remove it yourself.` - ); - } - }, - async overrideDuplicates(req) { const overrideLocale = self.apos.launder.boolean(req.body.overrideLocale); const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); @@ -458,7 +437,7 @@ module.exports = self => { throw self.apos.error(`invalid format "${formatLabel}"`); } - const { docs, attachmentsInfo } = await format.getFilesData(exportPath, docIds); + const { docs, attachmentsInfo } = await format.import(exportPath, docIds); const differentDocsLocale = self.getFirstDifferentLocale(req, docs); const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1; @@ -521,6 +500,21 @@ module.exports = self => { } }, + async setExportPathId(path) { + const id = self.apos.util.generateId(); + await self.apos.cache.set('exportPaths', id, path, 86400); + await self.apos.cache.set('exportPathIds', path, id, 86400); + return id; + }, + + async getExportPathById(id) { + return self.apos.cache.get('exportPaths', id); + }, + + async getExportPathId(path) { + return self.apos.cache.get('exportPathIds', path); + }, + async cleanExport(req) { const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); if (!exportPath) { @@ -538,23 +532,34 @@ module.exports = self => { self.apos.notification.dismiss(req, notificationId, 2000).catch(self.apos.util.error); } - await self.cleanFile(exportPath); - }, - - async setExportPathId(path) { - const id = self.apos.util.generateId(); - await self.apos.cache.set('exportPaths', id, path, 86400); - await self.apos.cache.set('exportPathIds', path, id, 86400); - return id; - }, - - async getExportPathById(id) { - return self.apos.cache.get('exportPaths', id); + await self.remove(exportPath); }, - async getExportPathId(path) { - return self.apos.cache.get('exportPathIds', path); + // FIXME: [Error: EPERM: operation not permitted, unlink '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9'] { + // errno: -1, + // code: 'EPERM', + // syscall: 'unlink', + // path: '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9' + // } + async remove(filepath) { + console.log('🚀 ~ remove ~ filepath:', filepath); + try { + const stat = await fsp.lstat(filepath); + console.log('🚀 ~ remove ~ stat:', stat); + if (stat.isDirectory()) { + await fsp.rm(filepath, { + recursive: true, + force: true + }); + } else { + await fsp.unlink(filepath); + } + } catch (err) { + console.trace(); + self.apos.util.error( + `Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.` + ); + } } - }; }; From 9a927762c729a0274b4230aa14a97608686a1f23 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 9 Apr 2024 16:48:40 +0200 Subject: [PATCH 10/42] import: fix remove error --- lib/methods/import.js | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index ec648ecc..5d54c56f 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -129,7 +129,7 @@ module.exports = self => { self.apos.util.error(error); }); - await self.remove(exportPath); + await self.removeExport(exportPath); return; } @@ -532,20 +532,12 @@ module.exports = self => { self.apos.notification.dismiss(req, notificationId, 2000).catch(self.apos.util.error); } - await self.remove(exportPath); + await self.removeExport(exportPath); }, - // FIXME: [Error: EPERM: operation not permitted, unlink '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9'] { - // errno: -1, - // code: 'EPERM', - // syscall: 'unlink', - // path: '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9' - // } - async remove(filepath) { - console.log('🚀 ~ remove ~ filepath:', filepath); + async removeExport(filepath) { try { const stat = await fsp.lstat(filepath); - console.log('🚀 ~ remove ~ stat:', stat); if (stat.isDirectory()) { await fsp.rm(filepath, { recursive: true, From 198950d7d33beda56575bb267a42035215cff81d Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 9 Apr 2024 16:56:33 +0200 Subject: [PATCH 11/42] add logs and clean existing ones --- lib/formats/csv.js | 21 +++++++----- lib/formats/gzip.js | 79 +++++++++++++++++++++++-------------------- lib/methods/export.js | 12 ++++--- lib/methods/import.js | 3 +- 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 1bf19e43..79e2ec56 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -48,13 +48,15 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { - console.log('IMPORT RETURNS:'); - console.dir({ - docs: !docIds - ? docs - : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), - exportPath - }, { depth: 9 }); + console.info(`[csv] docs read from ${exportPath}`); + + // console.log('IMPORT RETURNS:'); + // console.dir({ + // docs: !docIds + // ? docs + // : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + // exportPath + // }, { depth: 9 }); resolve({ docs: !docIds ? docs @@ -89,7 +91,10 @@ module.exports = { return new Promise((resolve, reject) => { stringifier.on('error', reject); writer.on('error', reject); - writer.on('finish', resolve); + writer.on('finish', () => { + console.info(`[csv] export file written to ${exportPath}`); + resolve(); + }); }); } }; diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 260ad445..44098456 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -18,40 +18,48 @@ module.exports = { // importVersions: [ 'draft', 'published' ], // exportVersions: [ 'draft', 'published' ], async import(exportPath, { docIds } = {}) { - console.log('🚀 ~ import ~ exportPath:', exportPath); - console.log('🚀 ~ import ~ docIds:', docIds); - console.log('this.allowedExtension', this.allowedExtension); // If the given path is actually the archive, we first need to extract it. // Then we no longer need the archive file, so we remove it. if (exportPath.endsWith(this.allowedExtension)) { const filepath = exportPath; exportPath = exportPath.replace(this.allowedExtension, ''); + + console.info(`[gzip] extracting ${filepath} into ${exportPath}`); await extract(filepath, exportPath); - await remove(filepath); + console.info(`[gzip] removing ${filepath}`); + await remove(filepath); } - console.log('🚀 ~ import ~ exportPath:', exportPath); - const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); - const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); - console.log('IMPORT RETURNS:'); - console.dir({ - docs: !docIds - ? EJSON.parse(docs) - : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - attachment, - file: { - name: `${attachment.name}.${attachment.extension}`, - path: path.join( - exportPath, - 'attachments', - `${attachment._id}-${attachment.name}.${attachment.extension}` - ) - } - })), - exportPath - }, { depth: 9 }); + const docsPath = path.join(exportPath, 'aposDocs.json'); + const attachmentsPath = path.join(exportPath, 'aposAttachments.json'); + const attachmentFilesPath = path.join(exportPath, 'attachments'); + + const docs = await fsp.readFile(docsPath); + const attachments = await fsp.readFile(attachmentsPath); + + // console.log('IMPORT RETURNS:'); + // console.dir({ + // docs: !docIds + // ? EJSON.parse(docs) + // : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + // attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + // attachment, + // file: { + // name: `${attachment.name}.${attachment.extension}`, + // path: path.join( + // exportPath, + // 'attachments', + // `${attachment._id}-${attachment.name}.${attachment.extension}` + // ) + // } + // })), + // exportPath + // }, { depth: 9 }); + + console.info(`[gzip] docs read from ${docsPath}`); + console.info(`[gzip] attachments read from ${attachmentsPath}`); + console.info(`[gzip] attachment files read from ${attachmentFilesPath}`); return { docs: !docIds @@ -61,11 +69,7 @@ module.exports = { attachment, file: { name: `${attachment.name}.${attachment.extension}`, - path: path.join( - exportPath, - 'attachments', - `${attachment._id}-${attachment.name}.${attachment.extension}` - ) + path: path.join(attachmentFilesPath, `${attachment._id}-${attachment.name}.${attachment.extension}`) } })), exportPath @@ -104,10 +108,12 @@ module.exports = { pack.on('error', reject); writeStream.on('finish', () => { + console.info(`[gzip] export file written to ${filepath}`); resolve(result); }); for (const [ filename, content ] of Object.entries(data.json || {})) { + console.info(`[gzip] adding ${filename} to the tarball`); addTarEntry(pack, { name: filename }, content).catch(reject); } @@ -116,9 +122,9 @@ module.exports = { type: 'directory' }) .then(() => { - console.log('attachments'); processAttachments(data.attachments, async (temp, name, size) => { - console.log('callback', { temp, name, size }); + console.info(`[gzip] adding attachments/${name} to the tarball`); + const readStream = fs.createReadStream(temp); const entryStream = pack.entry({ name: `attachments/${name}`, @@ -138,9 +144,6 @@ module.exports = { }; async function extract(filepath, exportPath) { - console.log('🚀 ~ extract ~ filepath:', filepath); - console.log('🚀 ~ extract ~ exportPath:', exportPath); - if (!fs.existsSync(exportPath)) { await fsp.mkdir(exportPath); } @@ -173,12 +176,14 @@ async function extract(filepath, exportPath) { }); } +// This independent function is designed for file removal. +// Avoid invoking `self.remove` within this script, +// as it should remain separate from the apos context. async function remove(filepath) { - console.log('🚀 ~ remove ~ filepath:', filepath); try { await fsp.unlink(filepath); } catch (error) { - self.apos.util.error(error); + console.error(error); } } diff --git a/lib/methods/export.js b/lib/methods/export.js index 72a8a382..79ff079e 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -297,6 +297,7 @@ module.exports = self => { const downloadPath = path.join('/exports', filename); const downloadUrl = `${self.apos.attachment.uploadfs.getUrl()}${downloadPath}`; const copyIn = util.promisify(self.apos.attachment.uploadfs.copyIn); + console.info(`[export] copying ${filepath} to ${self.apos.rootDir}/public/uploads${downloadPath}`); try { await copyIn(filepath, downloadPath); } catch (error) { @@ -341,7 +342,6 @@ module.exports = self => { async processAttachments(attachments, addAttachment) { let attachmentError = false; - console.log('🚀 ~ processAttachments ~ attachments:', attachments); const copyOut = Promise.promisify(self.apos.attachment.uploadfs.copyOut); await Promise.map(Object.entries(attachments), processAttachment, { @@ -351,12 +351,11 @@ module.exports = self => { return { attachmentError }; async function processAttachment([ name, url ]) { - console.log('🚀 ~ processAttachment ~ [ name, url ]:', [ name, url ]); const temp = self.apos.attachment.uploadfs.getTempPath() + '/' + self.apos.util.generateId(); + console.info(`[export] processing attachment ${name} temporarily stored in ${temp}`); try { await copyOut(url, temp); const { size } = await fsp.stat(temp); - console.log('🚀 ~ processAttachment ~ size:', size); // Looking at the source code, the tar-stream module // probably doesn't protect against two input streams // pushing mishmashed bytes into the tarball at the @@ -365,7 +364,6 @@ module.exports = self => { // than this operation await self.apos.lock.withLock('import-export-copy-out', async () => { // Add attachment into the specific format - console.log('addAttachment', { temp, name, size }); await addAttachment(temp, name, size); }); } catch (e) { @@ -378,6 +376,7 @@ module.exports = self => { }, async remove(filepath) { + console.info(`[export] removing ${filepath}`); try { await fsp.unlink(filepath); } catch (error) { @@ -387,13 +386,16 @@ module.exports = self => { // Report is available for 10 minutes by default removeFromUploadFs(downloadPath, expiration) { + const ms = expiration || 1000 * 60 * 10; + console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); setTimeout(() => { + console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); self.apos.attachment.uploadfs.remove(downloadPath, error => { if (error) { self.apos.util.error(error); } }); - }, expiration || 1000 * 60 * 10); + }, ms); } }; }; diff --git a/lib/methods/import.js b/lib/methods/import.js index 5d54c56f..1017005b 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -423,7 +423,6 @@ module.exports = self => { const importedAttachments = self.apos.launder.strings(req.body.importedAttachments); const formatLabel = self.apos.launder.string(req.body.formatLabel); const failedIds = []; - // console.log('🚀 ~ overrideDuplicates ~ formatLabel:', formatLabel); const jobManager = self.apos.modules['@apostrophecms/job']; const job = await jobManager.db.findOne({ _id: jobId }); @@ -516,6 +515,7 @@ module.exports = self => { }, async cleanExport(req) { + console.info('[import] cleaning export...'); const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); if (!exportPath) { throw self.apos.error.invalid('no such export path'); @@ -546,6 +546,7 @@ module.exports = self => { } else { await fsp.unlink(filepath); } + console.info(`[import] export path ${filepath} has been removed`); } catch (err) { console.trace(); self.apos.util.error( From b90b8bd91cd6eb93a2a30199a98941423c35ff4c Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 9 Apr 2024 18:38:45 +0200 Subject: [PATCH 12/42] separation of concerns: stop filtering docs in format script --- lib/formats/csv.js | 26 ++++++-------------- lib/formats/gzip.js | 57 ++++++++++++++++--------------------------- lib/methods/import.js | 18 ++++++++++---- 3 files changed, 41 insertions(+), 60 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 79e2ec56..7ba959d0 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -11,8 +11,8 @@ module.exports = { // TODO: remove these? because it really messes with the duplicated docs detection system // importVersions: [ 'draft' ], // exportVersions: [ 'published' ], - async import(exportPath, { docIds } = {}) { - const reader = fs.createReadStream(exportPath); + async import(filepath) { + const reader = fs.createReadStream(filepath); const parser = reader .pipe( parse({ @@ -48,26 +48,14 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { - console.info(`[csv] docs read from ${exportPath}`); + console.info(`[csv] docs read from ${filepath}`); - // console.log('IMPORT RETURNS:'); - // console.dir({ - // docs: !docIds - // ? docs - // : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), - // exportPath - // }, { depth: 9 }); - resolve({ - docs: !docIds - ? docs - : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), - exportPath - }); + resolve({ docs }); }); }); }, - async export(exportPath, { docs }) { - const writer = fs.createWriteStream(exportPath); + async export(filepath, { docs }) { + const writer = fs.createWriteStream(filepath); const stringifier = stringify({ header: true, columns: getColumnsNames(docs), @@ -92,7 +80,7 @@ module.exports = { stringifier.on('error', reject); writer.on('error', reject); writer.on('finish', () => { - console.info(`[csv] export file written to ${exportPath}`); + console.info(`[csv] export file written to ${filepath}`); resolve(); }); }); diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 44098456..d6e32145 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -17,12 +17,13 @@ module.exports = { includeAttachments: true, // importVersions: [ 'draft', 'published' ], // exportVersions: [ 'draft', 'published' ], - async import(exportPath, { docIds } = {}) { + async import(filepath) { + let exportPath = filepath; + // If the given path is actually the archive, we first need to extract it. // Then we no longer need the archive file, so we remove it. - if (exportPath.endsWith(this.allowedExtension)) { - const filepath = exportPath; - exportPath = exportPath.replace(this.allowedExtension, ''); + if (filepath.endsWith(this.allowedExtension)) { + exportPath = filepath.replace(this.allowedExtension, ''); console.info(`[gzip] extracting ${filepath} into ${exportPath}`); await extract(filepath, exportPath); @@ -35,43 +36,27 @@ module.exports = { const attachmentsPath = path.join(exportPath, 'aposAttachments.json'); const attachmentFilesPath = path.join(exportPath, 'attachments'); + console.info(`[gzip] reading docs from ${docsPath}`); + console.info(`[gzip] reading attachments from ${attachmentsPath}`); + console.info(`[gzip] reading attachment files from ${attachmentFilesPath}`); + const docs = await fsp.readFile(docsPath); const attachments = await fsp.readFile(attachmentsPath); - // console.log('IMPORT RETURNS:'); - // console.dir({ - // docs: !docIds - // ? EJSON.parse(docs) - // : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - // attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - // attachment, - // file: { - // name: `${attachment.name}.${attachment.extension}`, - // path: path.join( - // exportPath, - // 'attachments', - // `${attachment._id}-${attachment.name}.${attachment.extension}` - // ) - // } - // })), - // exportPath - // }, { depth: 9 }); - - console.info(`[gzip] docs read from ${docsPath}`); - console.info(`[gzip] attachments read from ${attachmentsPath}`); - console.info(`[gzip] attachment files read from ${attachmentFilesPath}`); + const parsedDocs = EJSON.parse(docs); + const parsedAttachments = EJSON.parse(attachments); + + const attachmentsInfo = parsedAttachments.map(attachment => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join(attachmentFilesPath, `${attachment._id}-${attachment.name}.${attachment.extension}`) + } + })); return { - docs: !docIds - ? EJSON.parse(docs) - : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - attachment, - file: { - name: `${attachment.name}.${attachment.extension}`, - path: path.join(attachmentFilesPath, `${attachment._id}-${attachment.name}.${attachment.extension}`) - } - })), + docs: parsedDocs, + attachmentsInfo, exportPath }; }, diff --git a/lib/methods/import.js b/lib/methods/import.js index 1017005b..7b80d21a 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -35,10 +35,16 @@ module.exports = self => { attachmentsInfo = [] } = await format.import(exportPath)); } else { + // If no export path is found, it means the user is importing a file for the first time. + // A format might need to extract the file content into a temporary directory, so we need to + // store the path to be able to clean it up later. + // By default, the export path is the same as the file path. + // In the case of an archive, the export path won't be the same as the file path because + // it will be extracted into a temporary directory. That directory will be the export path. ({ docs = [], attachmentsInfo = [], - exportPath + exportPath = file.path } = await format.import(file.path)); await self.setExportPathId(exportPath); @@ -436,19 +442,21 @@ module.exports = self => { throw self.apos.error(`invalid format "${formatLabel}"`); } - const { docs, attachmentsInfo } = await format.import(exportPath, docIds); + const { docs, attachmentsInfo } = await format.import(exportPath); - const differentDocsLocale = self.getFirstDifferentLocale(req, docs); + const filterDocs = docs.filter(({ aposDocId }) => docIds.includes(aposDocId)); + + const differentDocsLocale = self.getFirstDifferentLocale(req, filterDocs); const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1; // Re-write locale if `overrideLocale` param is passed-on from the import process // (i.e if the user chose "Yes") // or re-write locale automatically on a single-locale site if (differentDocsLocale && (!siteHasMultipleLocales || overrideLocale)) { - self.rewriteDocsWithCurrentLocale(req, docs); + self.rewriteDocsWithCurrentLocale(req, filterDocs); } - for (const doc of docs) { + for (const doc of filterDocs) { try { const attachmentsToOverride = self.getRelatedDocsFromSchema(req, { doc, From 94146870f6cfaf3e79deac586f243c5d4969e248 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 9 Apr 2024 18:39:34 +0200 Subject: [PATCH 13/42] add xslx format --- README.md | 4 ++++ lib/formats/index.js | 7 +++++-- lib/formats/xlsx.js | 31 +++++++++++++++++++++++++++++++ package.json | 3 ++- 4 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 lib/formats/xlsx.js diff --git a/README.md b/README.md index 14b3cb66..4fa0140b 100644 --- a/README.md +++ b/README.md @@ -147,3 +147,7 @@ Exported documents maintain their locale settings. If the locale during import d If multiple locales are set up, the user will be prompted to choose between canceling the import or proceeding with it. ![Screenshot highlighting the confirm modal letting the user choose between aborting on continuing the import when the docs locale is different from the site one.](https://static.apostrophecms.com/apostrophecms/import-export/images/different-locale-modal.png) + +## How to add a new format? + +TODO: diff --git a/lib/formats/index.js b/lib/formats/index.js index 6c1073fe..909feb77 100644 --- a/lib/formats/index.js +++ b/lib/formats/index.js @@ -1,7 +1,10 @@ -const csv = require('./csv'); const gzip = require('./gzip'); +const csv = require('./csv'); +const xlsx = require('./xlsx'); +// Keep gzip at the top of the list so it's the default format module.exports = { + gzip, csv, - gzip + xlsx }; diff --git a/lib/formats/xlsx.js b/lib/formats/xlsx.js new file mode 100644 index 00000000..7929bcfe --- /dev/null +++ b/lib/formats/xlsx.js @@ -0,0 +1,31 @@ +const XLSX = require('xlsx'); + +const SHEET_NAME = 'docs'; + +module.exports = { + label: 'XLSX', + extension: '.xlsx', + allowedExtension: '.xlsx', + allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], + includeAttachments: false, + async import(filepath) { + const workbook = XLSX.readFile(filepath); + + const sheet = workbook.Sheets[SHEET_NAME]; + const docs = XLSX.utils.sheet_to_json(sheet); + + // TODO: handle dates, numbers and floats, and objects and arrays + console.log('🚀 ~ import ~ docs:', docs); + + return { docs }; + }, + async export(filepath, { docs }) { + const workbook = XLSX.utils.book_new(); + const worksheet = XLSX.utils.json_to_sheet(docs); + + XLSX.utils.book_append_sheet(workbook, worksheet, SHEET_NAME); + XLSX.writeFile(workbook, filepath); + + console.info(`[xlsx] docs and attachments written to ${filepath}`); + } +}; diff --git a/package.json b/package.json index c7a202c8..71dadc81 100644 --- a/package.json +++ b/package.json @@ -37,6 +37,7 @@ "csv-stringify": "^6.4.6", "dayjs": "^1.9.8", "lodash": "^4.17.21", - "tar-stream": "^3.1.6" + "tar-stream": "^3.1.6", + "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" } } From 6521d2d424a543d2067a34d9e2d7571ed5fb1514 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 9 Apr 2024 18:39:50 +0200 Subject: [PATCH 14/42] fix select line-height --- ui/apos/components/AposExportModal.vue | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ui/apos/components/AposExportModal.vue b/ui/apos/components/AposExportModal.vue index 325af13e..4916a449 100644 --- a/ui/apos/components/AposExportModal.vue +++ b/ui/apos/components/AposExportModal.vue @@ -330,6 +330,11 @@ export default { display: flex; } +:deep(.apos-input--select) { + padding-right: 40px; + line-height: var(--a-line-tall); +} + .apos-export__heading { @include type-title; line-height: var(--a-line-tall); From b59c99fde676b2dc92311f66c7ccd70920d820d1 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 10 Apr 2024 10:48:13 +0200 Subject: [PATCH 15/42] better use of options and default values --- lib/formats/csv.js | 4 ---- lib/formats/gzip.js | 2 -- lib/formats/xlsx.js | 1 - lib/methods/export.js | 10 +------- lib/methods/import.js | 54 +++++++++++++++++++------------------------ 5 files changed, 25 insertions(+), 46 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 7ba959d0..de10cee1 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -7,10 +7,6 @@ module.exports = { extension: '.csv', allowedExtension: '.csv', allowedTypes: [ 'text/csv' ], - includeAttachments: false, - // TODO: remove these? because it really messes with the duplicated docs detection system - // importVersions: [ 'draft' ], - // exportVersions: [ 'published' ], async import(filepath) { const reader = fs.createReadStream(filepath); const parser = reader diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index d6e32145..e8267354 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -15,8 +15,6 @@ module.exports = { 'application/x-gzip' ], includeAttachments: true, - // importVersions: [ 'draft', 'published' ], - // exportVersions: [ 'draft', 'published' ], async import(filepath) { let exportPath = filepath; diff --git a/lib/formats/xlsx.js b/lib/formats/xlsx.js index 7929bcfe..75e51e75 100644 --- a/lib/formats/xlsx.js +++ b/lib/formats/xlsx.js @@ -7,7 +7,6 @@ module.exports = { extension: '.xlsx', allowedExtension: '.xlsx', allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], - includeAttachments: false, async import(filepath) { const workbook = XLSX.readFile(filepath); diff --git a/lib/methods/export.js b/lib/methods/export.js index 79ff079e..dc8bdc6d 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -153,14 +153,6 @@ module.exports = self => { return docs.filter(doc => self.canExport(req, doc.type)); }, - shouldExportDraft(format) { - return !format.exportVersions?.length || format.exportVersions.includes('draft'); - }, - - shouldExportPublished(format) { - return !format.exportVersions?.length || format.exportVersions.includes('published'); - }, - // Add the published version ID next to each draft ID, // so we always get both the draft and the published ID. // If somehow published IDs are sent from the frontend, @@ -276,7 +268,7 @@ module.exports = self => { try { const result = await format.export(filepath, data, self.processAttachments); - if (format.includeAttachments && result?.attachmentError) { + if (result?.attachmentError) { await self.apos.notify(req, 'aposImportExport:exportAttachmentError', { interpolate: { format: format.label }, icon: 'alert-circle-icon', diff --git a/lib/methods/import.js b/lib/methods/import.js index 7b80d21a..623c242d 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -31,7 +31,7 @@ module.exports = self => { try { if (exportPath) { ({ - docs = [], + docs, attachmentsInfo = [] } = await format.import(exportPath)); } else { @@ -42,7 +42,7 @@ module.exports = self => { // In the case of an archive, the export path won't be the same as the file path because // it will be extracted into a temporary directory. That directory will be the export path. ({ - docs = [], + docs, attachmentsInfo = [], exportPath = file.path } = await format.import(file.path)); @@ -370,14 +370,6 @@ module.exports = self => { return manager[method](_req, doc, { setModified: false }); }, - shouldImportDraft(format) { - return !format.importVersions?.length || format.importVersions.includes('draft'); - }, - - shouldImportPublished(format) { - return !format.importVersions?.length || format.importVersions.includes('published'); - }, - canImport(req, docType) { return self.canImportOrExport(req, docType, 'import'); }, @@ -442,9 +434,9 @@ module.exports = self => { throw self.apos.error(`invalid format "${formatLabel}"`); } - const { docs, attachmentsInfo } = await format.import(exportPath); + const { docs, attachmentsInfo = [] } = await format.import(exportPath); - const filterDocs = docs.filter(({ aposDocId }) => docIds.includes(aposDocId)); + const filterDocs = docs.filter(doc => docIds.includes(doc.aposDocId)); const differentDocsLocale = self.getFirstDifferentLocale(req, filterDocs); const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1; @@ -458,12 +450,6 @@ module.exports = self => { for (const doc of filterDocs) { try { - const attachmentsToOverride = self.getRelatedDocsFromSchema(req, { - doc, - schema: self.apos.modules[doc.type].schema, - type: 'attachment' - }); - await self.insertOrUpdateDoc(req, { doc, method: 'update', @@ -472,19 +458,27 @@ module.exports = self => { jobManager.success(job); - for (const { _id } of attachmentsToOverride) { - if (importedAttachments.includes(_id)) { - continue; - } - const attachmentInfo = attachmentsInfo - .find(({ attachment }) => attachment._id === _id); + if (attachmentsInfo.length) { + const attachmentsToOverride = self.getRelatedDocsFromSchema(req, { + doc, + schema: self.apos.modules[doc.type].schema, + type: 'attachment' + }); - try { - await self.insertOrUpdateAttachment(req, { attachmentInfo }); - jobManager.success(job); - importedAttachments.push(_id); - } catch (err) { - jobManager.failure(job); + for (const { _id } of attachmentsToOverride) { + if (importedAttachments.includes(_id)) { + continue; + } + const attachmentInfo = attachmentsInfo + .find(({ attachment }) => attachment._id === _id); + + try { + await self.insertOrUpdateAttachment(req, { attachmentInfo }); + jobManager.success(job); + importedAttachments.push(_id); + } catch (err) { + jobManager.failure(job); + } } } } catch (err) { From a8965cf0b1884513fc0c654dae8cdb9702a16e5b Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 10 Apr 2024 11:27:15 +0200 Subject: [PATCH 16/42] clean csv options --- lib/formats/csv.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index de10cee1..527b699d 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -19,9 +19,6 @@ module.exports = { return value; } - // console.log(context.column, value, typeof value); - // TODO: need to handle Dates, number and floats? - try { return JSON.parse(value); } catch { @@ -45,7 +42,6 @@ module.exports = { parser.on('error', reject); parser.on('end', () => { console.info(`[csv] docs read from ${filepath}`); - resolve({ docs }); }); }); @@ -55,7 +51,6 @@ module.exports = { const stringifier = stringify({ header: true, columns: getColumnsNames(docs), - // quoted_string: true, cast: { date(value) { return value.toISOString(); From 4a7247e01b2c116d0fd4d969c51840443f1fb08e Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 10 Apr 2024 16:16:47 +0200 Subject: [PATCH 17/42] csv and xlsx parsing - wip --- lib/formats/csv.js | 4 ++++ lib/formats/xlsx.js | 37 +++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 527b699d..18106a2e 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -54,6 +54,10 @@ module.exports = { cast: { date(value) { return value.toISOString(); + }, + boolean(value) { + // TODO: check + return value ? 'true' : 'false'; } } }); diff --git a/lib/formats/xlsx.js b/lib/formats/xlsx.js index 75e51e75..1b98bbd0 100644 --- a/lib/formats/xlsx.js +++ b/lib/formats/xlsx.js @@ -13,18 +13,43 @@ module.exports = { const sheet = workbook.Sheets[SHEET_NAME]; const docs = XLSX.utils.sheet_to_json(sheet); - // TODO: handle dates, numbers and floats, and objects and arrays - console.log('🚀 ~ import ~ docs:', docs); - - return { docs }; + return { docs: docs.map(parse) }; }, async export(filepath, { docs }) { const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.json_to_sheet(docs); + const worksheet = XLSX.utils.json_to_sheet( + docs.map(stringify) + ); XLSX.utils.book_append_sheet(workbook, worksheet, SHEET_NAME); - XLSX.writeFile(workbook, filepath); + XLSX.writeFile(workbook, filepath, { compression: true }); console.info(`[xlsx] docs and attachments written to ${filepath}`); } }; + +function stringify(doc) { + const object = {}; + for (const key in doc) { + object[key] = typeof doc[key] === 'object' + ? JSON.stringify(doc[key]) + : doc[key]; + } + return object; +} + +function parse(doc) { + const object = {}; + for (const key in doc) { + if (!doc[key]) { + // Avoid setting empty values as empty strings + continue; + } + try { + object[key] = JSON.parse(doc[key]); + } catch { + object[key] = doc[key]; + } + } + return object; +} From cb952c44c76861fa38e22c5c931b17e9a45e7a5f Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 10 Apr 2024 17:39:51 +0200 Subject: [PATCH 18/42] clean --- lib/methods/import.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index 623c242d..432425c7 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -115,8 +115,7 @@ module.exports = self => { if (!duplicatedDocs.length) { await reporting.end(); - const notifMsg = `aposImportExport:${ - failedIds.length ? 'importFailedForSome' : 'importSucceed'}`; + const notifMsg = `aposImportExport:${failedIds.length ? 'importFailedForSome' : 'importSucceed'}`; await self.apos.notify(req, notifMsg, { interpolate: { @@ -130,10 +129,9 @@ module.exports = self => { } }); - self.apos.notification.dismiss(req, notificationId, 2000) - .catch((error) => { - self.apos.util.error(error); - }); + self.apos.notification + .dismiss(req, notificationId, 2000) + .catch(self.apos.util.error); await self.removeExport(exportPath); return; From 24704ef24c447f54efe44a726f440fcf323ed1c6 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 10 Apr 2024 18:59:36 +0200 Subject: [PATCH 19/42] thanks to cypress, fix a bug when overriding locales and clarify code --- lib/methods/import.js | 49 +++++++++++++++++++++---------------------- ui/apos/apps/index.js | 3 ++- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index 432425c7..ec987956 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -9,38 +9,36 @@ module.exports = self => { } const { file } = req.files || {}; - - if (!file) { - throw self.apos.error('invalid'); - } - - const format = Object - .values(self.formats) - .find((format) => format.allowedTypes.includes(file.type)); - - if (!format) { - throw self.apos.error('invalid'); - } - const overrideLocale = self.apos.launder.boolean(req.body.overrideLocale); + const formatLabel = self.apos.launder.string(req.body.formatLabel); let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); - + let format; let docs; let attachmentsInfo; try { - if (exportPath) { - ({ - docs, - attachmentsInfo = [] - } = await format.import(exportPath)); + if (overrideLocale) { + if (!formatLabel) { + throw self.apos.error('invalid: no `formatLabel` provided'); + } + if (!exportPath) { + throw self.apos.error('invalid: no `exportPath` provided'); + } + + format = Object + .values(self.formats) + .find(format => format.label === formatLabel); + + ({ docs, attachmentsInfo = [] } = await format.import(exportPath)); } else { - // If no export path is found, it means the user is importing a file for the first time. - // A format might need to extract the file content into a temporary directory, so we need to - // store the path to be able to clean it up later. - // By default, the export path is the same as the file path. - // In the case of an archive, the export path won't be the same as the file path because - // it will be extracted into a temporary directory. That directory will be the export path. + if (!file) { + throw new Error('invalid: no file provided'); + } + + format = Object + .values(self.formats) + .find(format => format.allowedTypes.includes(file.type)); + ({ docs, attachmentsInfo = [], @@ -73,6 +71,7 @@ module.exports = self => { data: { moduleName, exportPathId: await self.getExportPathId(exportPath), + formatLabel: format.label, content: { heading: req.t('aposImportExport:importWithCurrentLocaleHeading'), description: req.t('aposImportExport:importWithCurrentLocaleDescription', { diff --git a/ui/apos/apps/index.js b/ui/apos/apps/index.js index b30ed73d..54633d00 100644 --- a/ui/apos/apps/index.js +++ b/ui/apos/apps/index.js @@ -36,7 +36,8 @@ export default () => { await apos.http.post(`${moduleAction}/import-export-import`, { body: { overrideLocale: true, - exportPathId: event.exportPathId + exportPathId: event.exportPathId, + formatLabel: event.formatLabel } }); } catch (error) { From 86df2166b5424a0f8dfaac8df0e8dbfd41fb8db6 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Thu, 11 Apr 2024 17:26:10 +0200 Subject: [PATCH 20/42] refactor remove functions --- lib/methods/export.js | 9 --------- lib/methods/import.js | 24 ++---------------------- lib/methods/index.js | 22 ++++++++++++++++++++++ 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index dc8bdc6d..d4e683b9 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -367,15 +367,6 @@ module.exports = self => { } }, - async remove(filepath) { - console.info(`[export] removing ${filepath}`); - try { - await fsp.unlink(filepath); - } catch (error) { - self.apos.util.error(error); - } - }, - // Report is available for 10 minutes by default removeFromUploadFs(downloadPath, expiration) { const ms = expiration || 1000 * 60 * 10; diff --git a/lib/methods/import.js b/lib/methods/import.js index ec987956..b7b8f458 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -132,7 +132,7 @@ module.exports = self => { .dismiss(req, notificationId, 2000) .catch(self.apos.util.error); - await self.removeExport(exportPath); + await self.remove(exportPath); return; } @@ -531,27 +531,7 @@ module.exports = self => { self.apos.notification.dismiss(req, notificationId, 2000).catch(self.apos.util.error); } - await self.removeExport(exportPath); - }, - - async removeExport(filepath) { - try { - const stat = await fsp.lstat(filepath); - if (stat.isDirectory()) { - await fsp.rm(filepath, { - recursive: true, - force: true - }); - } else { - await fsp.unlink(filepath); - } - console.info(`[import] export path ${filepath} has been removed`); - } catch (err) { - console.trace(); - self.apos.util.error( - `Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.` - ); - } + await self.remove(exportPath); } }; }; diff --git a/lib/methods/index.js b/lib/methods/index.js index f8d2e159..5c7840f4 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -1,3 +1,4 @@ +const fsp = require('node:fs/promises'); const importMethods = require('./import'); const exportMethods = require('./export'); @@ -34,6 +35,27 @@ module.exports = self => { return true; }, + + async remove(filepath) { + try { + const stat = await fsp.lstat(filepath); + if (stat.isDirectory()) { + await fsp.rm(filepath, { + recursive: true, + force: true + }); + } else { + await fsp.unlink(filepath); + } + console.info(`removed: ${filepath}`); + } catch (err) { + console.trace(); + self.apos.util.error( + `Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.` + ); + } + }, + ...importMethods(self), ...exportMethods(self) }; From d7e4882edebbf036b43e976179b4fee63ee0fe90 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Thu, 11 Apr 2024 17:27:09 +0200 Subject: [PATCH 21/42] hmmm --- lib/methods/import.js | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index b7b8f458..397b4ab0 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -1,4 +1,3 @@ -const fsp = require('node:fs/promises'); const { cloneDeep } = require('lodash'); module.exports = self => { From d348255eee880bea5b5f569ec170f235ea60edb7 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 16 Apr 2024 11:25:03 +0200 Subject: [PATCH 22/42] remove instantly from uploadfs (to discuss) --- lib/methods/export.js | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index d4e683b9..d4b85979 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -370,15 +370,22 @@ module.exports = self => { // Report is available for 10 minutes by default removeFromUploadFs(downloadPath, expiration) { const ms = expiration || 1000 * 60 * 10; - console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); - setTimeout(() => { + const remove = () => { console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); self.apos.attachment.uploadfs.remove(downloadPath, error => { if (error) { self.apos.util.error(error); } }); - }, ms); + }; + + if (process.env.CI) { + remove(); + return; + } + + console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); + setTimeout(remove, ms); } }; }; From 778fa4f85b3a1e4c2295d7182d05f510a50449fa Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 16 Apr 2024 11:31:35 +0200 Subject: [PATCH 23/42] Revert "remove instantly from uploadfs (to discuss)" This reverts commit d348255eee880bea5b5f569ec170f235ea60edb7. --- lib/methods/export.js | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/lib/methods/export.js b/lib/methods/export.js index d4b85979..d4e683b9 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -370,22 +370,15 @@ module.exports = self => { // Report is available for 10 minutes by default removeFromUploadFs(downloadPath, expiration) { const ms = expiration || 1000 * 60 * 10; - const remove = () => { + console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); + setTimeout(() => { console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs`); self.apos.attachment.uploadfs.remove(downloadPath, error => { if (error) { self.apos.util.error(error); } }); - }; - - if (process.env.CI) { - remove(); - return; - } - - console.info(`[export] removing ${self.apos.rootDir}/public/uploads${downloadPath} from uploadfs in ${ms / 1000 / 60} minutes`); - setTimeout(remove, ms); + }, ms); } }; }; From d14de399c3ae6fa3ffee5a2f63c941879b8d513c Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Tue, 16 Apr 2024 13:35:16 +0200 Subject: [PATCH 24/42] add doc to add new formats --- README.md | 122 +++++++++++++++++++++++++++++++++++++++++- lib/formats/gzip.js | 4 +- lib/methods/export.js | 2 +- 3 files changed, 124 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4fa0140b..541ed5f5 100644 --- a/README.md +++ b/README.md @@ -150,4 +150,124 @@ If multiple locales are set up, the user will be prompted to choose between canc ## How to add a new format? -TODO: +### Create a file for your format: + +Add your format under `lib/formats/.js` and export it in l`ib/formats/index.js`. + +**Simple example** (for a single file without attachment files): + +```js +// lib/formats/ods.js +module.exports = { + label: 'ODS', + extension: '.ods', + allowedExtension: '.ods', + allowedTypes: [ 'application/vnd.oasis.opendocument.spreadsheet' ], + async import(filepath) { + // Read `filepath` using `fs.createReadStream` + // or any reader provided by a third-party library + + // Return parsed docs as an array + return { docs }; + }, + async export(filepath, { docs }) { + // Write `docs` into `filepath` using `fs.createWriteStream` + // or any writer provided by a third-party library + } +}; +``` + +**Note**: The `import` and `export` functions should remain agnostic of any apostrophe logic. + +```js +// lib/formats/index.js +const ods = require('./ods'); + +module.exports = { + // ... + ods +}; +``` + +### For formats with attachment files: + +If you want to add a format that includes attachment files such as an archive, you can enable the `includeAttachments` option and utilize extra arguments provided in the `import` and `export` functions. + +**Advanced example**: + +```js +// lib/formats/zip.js +module.exports = { + label: 'ZIP', + extension: '.zip', + allowedExtension: '.zip', + allowedTypes: [ + 'application/zip', + 'application/x-zip', + 'application/x-zip-compressed' + ], + includeAttachments: true, + async import(filepath) { + let exportPath = filepath; + + // If the given path is the archive, we first need to extract it + // and define `exportPath` to the extracted folder, not the archive + if (filepath.endsWith(this.allowedExtension)) { + exportPath = filepath.replace(this.allowedExtension, ''); + + // Use format-specif extraction + await extract(filepath, exportPath); + await fsp.unlink(filepath); + } + + // Read docs and attachments from `exportPath` + // given that they are stored in aposDocs.json and aposAttachments.json files: + const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); + const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); + const parsedDocs = EJSON.parse(docs); + const parsedAttachments = EJSON.parse(attachments); + + // Add the attachment names and their path where they are going to be written to + const attachmentsInfo = parsedAttachments.map(attachment => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join(exportPath, 'attachments', `${attachment._id}-${attachment.name}.${attachment.extension}`) + } + })); + + // Return parsed docs as an array, attachments with their extra files info + // and `exportPath` since it we need to inform the caller where the extracted data is: + return { + docs: parsedDocs, + attachmentsInfo, + exportPath + }; + }, + async export( + filepath, + { + docs, + attachments = [], + attachmentUrls = {} + }, + processAttachments + ) { + // Store the docs and attachments into `aposDocs.json` and `aposAttachments.json` files + // and add them to the archive + + // Create a `attachments/` directory in the archive and store the attachment files inside it: + const addAttachment = async (attachmentPath, name, size) => { + // Read attachment from `attachmentPath` + // and store it into `attachments/` inside the archive + } + const { attachmentError } = await processAttachments(attachmentUrls, addAttachment); + + // Write the archive that contains `aposDocs.json`, `aposAttachments.json` and `attachments/` + // into `filepath` using `fs.createWriteStream` or any writer provided by a third-party library + + // Return potential attachment processing error so that the caller is aware of it: + return { attachmentError }; + } +}; +``` diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index e8267354..b1204ca8 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -105,10 +105,10 @@ module.exports = { type: 'directory' }) .then(() => { - processAttachments(data.attachments, async (temp, name, size) => { + processAttachments(data.attachments, async (attachmentPath, name, size) => { console.info(`[gzip] adding attachments/${name} to the tarball`); - const readStream = fs.createReadStream(temp); + const readStream = fs.createReadStream(attachmentPath); const entryStream = pack.entry({ name: `attachments/${name}`, size diff --git a/lib/methods/export.js b/lib/methods/export.js index d4e683b9..97472ec4 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -348,7 +348,7 @@ module.exports = self => { try { await copyOut(url, temp); const { size } = await fsp.stat(temp); - // Looking at the source code, the tar-stream module + // Looking at the source code, the tar-stream module (when using the `gzip` format) // probably doesn't protect against two input streams // pushing mishmashed bytes into the tarball at the // same time, so stream in just one at a time. We still From 132616d2e4a4900ee253798314b1902cc542435a Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 17 Apr 2024 17:41:37 +0200 Subject: [PATCH 25/42] revert to input and output --- README.md | 8 ++++---- lib/formats/csv.js | 4 ++-- lib/formats/gzip.js | 4 ++-- lib/formats/xlsx.js | 4 ++-- lib/methods/export.js | 2 +- lib/methods/import.js | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 541ed5f5..9a0a8331 100644 --- a/README.md +++ b/README.md @@ -163,14 +163,14 @@ module.exports = { extension: '.ods', allowedExtension: '.ods', allowedTypes: [ 'application/vnd.oasis.opendocument.spreadsheet' ], - async import(filepath) { + async input(filepath) { // Read `filepath` using `fs.createReadStream` // or any reader provided by a third-party library // Return parsed docs as an array return { docs }; }, - async export(filepath, { docs }) { + async output(filepath, { docs }) { // Write `docs` into `filepath` using `fs.createWriteStream` // or any writer provided by a third-party library } @@ -207,7 +207,7 @@ module.exports = { 'application/x-zip-compressed' ], includeAttachments: true, - async import(filepath) { + async input(filepath) { let exportPath = filepath; // If the given path is the archive, we first need to extract it @@ -244,7 +244,7 @@ module.exports = { exportPath }; }, - async export( + async output( filepath, { docs, diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 18106a2e..37ef417a 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -7,7 +7,7 @@ module.exports = { extension: '.csv', allowedExtension: '.csv', allowedTypes: [ 'text/csv' ], - async import(filepath) { + async input(filepath) { const reader = fs.createReadStream(filepath); const parser = reader .pipe( @@ -46,7 +46,7 @@ module.exports = { }); }); }, - async export(filepath, { docs }) { + async output(filepath, { docs }) { const writer = fs.createWriteStream(filepath); const stringifier = stringify({ header: true, diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index b1204ca8..250add83 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -15,7 +15,7 @@ module.exports = { 'application/x-gzip' ], includeAttachments: true, - async import(filepath) { + async input(filepath) { let exportPath = filepath; // If the given path is actually the archive, we first need to extract it. @@ -58,7 +58,7 @@ module.exports = { exportPath }; }, - async export( + async output( filepath, { docs, diff --git a/lib/formats/xlsx.js b/lib/formats/xlsx.js index 1b98bbd0..9c2d5d9a 100644 --- a/lib/formats/xlsx.js +++ b/lib/formats/xlsx.js @@ -7,7 +7,7 @@ module.exports = { extension: '.xlsx', allowedExtension: '.xlsx', allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], - async import(filepath) { + async input(filepath) { const workbook = XLSX.readFile(filepath); const sheet = workbook.Sheets[SHEET_NAME]; @@ -15,7 +15,7 @@ module.exports = { return { docs: docs.map(parse) }; }, - async export(filepath, { docs }) { + async output(filepath, { docs }) { const workbook = XLSX.utils.book_new(); const worksheet = XLSX.utils.json_to_sheet( docs.map(stringify) diff --git a/lib/methods/export.js b/lib/methods/export.js index 97472ec4..1264073d 100644 --- a/lib/methods/export.js +++ b/lib/methods/export.js @@ -266,7 +266,7 @@ module.exports = self => { const filepath = path.join(self.apos.attachment.uploadfs.getTempPath(), filename); try { - const result = await format.export(filepath, data, self.processAttachments); + const result = await format.output(filepath, data, self.processAttachments); if (result?.attachmentError) { await self.apos.notify(req, 'aposImportExport:exportAttachmentError', { diff --git a/lib/methods/import.js b/lib/methods/import.js index 397b4ab0..7ea49555 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -28,7 +28,7 @@ module.exports = self => { .values(self.formats) .find(format => format.label === formatLabel); - ({ docs, attachmentsInfo = [] } = await format.import(exportPath)); + ({ docs, attachmentsInfo = [] } = await format.input(exportPath)); } else { if (!file) { throw new Error('invalid: no file provided'); @@ -42,7 +42,7 @@ module.exports = self => { docs, attachmentsInfo = [], exportPath = file.path - } = await format.import(file.path)); + } = await format.input(file.path)); await self.setExportPathId(exportPath); } @@ -430,7 +430,7 @@ module.exports = self => { throw self.apos.error(`invalid format "${formatLabel}"`); } - const { docs, attachmentsInfo = [] } = await format.import(exportPath); + const { docs, attachmentsInfo = [] } = await format.input(exportPath); const filterDocs = docs.filter(doc => docIds.includes(doc.aposDocId)); From c2e6beefb2a6e632d418576da58727d1b5943343 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 22 Apr 2024 16:30:48 +0200 Subject: [PATCH 26/42] tests wip --- lib/formats/gzip.js | 2 ++ lib/formats/index.js | 2 +- lib/methods/import.js | 1 + test/index.js | 40 ++++++++++++++++++++-------------------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 250add83..7517db12 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -16,12 +16,14 @@ module.exports = { ], includeAttachments: true, async input(filepath) { + console.log('🚀 ~ input ~ filepath:', filepath); let exportPath = filepath; // If the given path is actually the archive, we first need to extract it. // Then we no longer need the archive file, so we remove it. if (filepath.endsWith(this.allowedExtension)) { exportPath = filepath.replace(this.allowedExtension, ''); + console.log('🚀 ~ input ~ exportPath:', exportPath); console.info(`[gzip] extracting ${filepath} into ${exportPath}`); await extract(filepath, exportPath); diff --git a/lib/formats/index.js b/lib/formats/index.js index 909feb77..89b1d50a 100644 --- a/lib/formats/index.js +++ b/lib/formats/index.js @@ -6,5 +6,5 @@ const xlsx = require('./xlsx'); module.exports = { gzip, csv, - xlsx + xlsx // TODO: make it a separate module to register the XLSX format, and make formats registerable: import-export-xlsx }; diff --git a/lib/methods/import.js b/lib/methods/import.js index 7ea49555..656e6ab4 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -33,6 +33,7 @@ module.exports = self => { if (!file) { throw new Error('invalid: no file provided'); } + console.log(file); format = Object .values(self.formats) diff --git a/test/index.js b/test/index.js index 639fe634..13b77a34 100644 --- a/test/index.js +++ b/test/index.js @@ -16,7 +16,7 @@ describe('@apostrophecms/import-export', function () { let mimeType; let piecesTgzPath; let pageTgzPath; - let cleanFile; + let remove; this.timeout(60000); @@ -38,14 +38,14 @@ describe('@apostrophecms/import-export', function () { importExportManager.removeExportFileFromUploadFs = () => {}; gzip = importExportManager.formats.gzip; mimeType = gzip.allowedTypes[0]; - cleanFile = importExportManager.cleanFile; - importExportManager.cleanFile = () => {}; + remove = importExportManager.remove; + importExportManager.remove = () => {}; await insertAdminUser(apos); await insertPieces(apos); }); - it('should generate a zip file for pieces without related documents', async function () { + it.only('should generate a zip file for pieces without related documents', async function () { const req = apos.task.getReq(); const articles = await apos.article.find(req).toArray(); const manager = apos.article; @@ -58,7 +58,7 @@ describe('@apostrophecms/import-export', function () { const { url } = await importExportManager.export(req, manager); const fileName = path.basename(url); - const exportPath = await gzip.input(path.join(exportsPath, fileName)); + const { exportPath } = await gzip.input(path.join(exportsPath, fileName)); const { docs, attachments, attachmentFiles @@ -75,11 +75,11 @@ describe('@apostrophecms/import-export', function () { attachmentFiles: [] }; - await cleanFile(exportPath); + await remove(exportPath); assert.deepEqual(actual, expected); }); - it('should generate a zip file for pieces with related documents', async function () { + it.only('should generate a zip file for pieces with related documents', async function () { const req = apos.task.getReq(); const articles = await apos.article.find(req).toArray(); const { _id: attachmentId } = await apos.attachment.db.findOne({ name: 'test-image' }); @@ -96,7 +96,7 @@ describe('@apostrophecms/import-export', function () { const fileName = path.basename(url); piecesTgzPath = path.join(exportsPath, fileName); - const exportPath = await gzip.input(piecesTgzPath); + const { exportPath } = await gzip.input(piecesTgzPath); const { docs, attachments, attachmentFiles @@ -150,11 +150,11 @@ describe('@apostrophecms/import-export', function () { attachmentFiles: [ `${attachmentId}-test-image.jpg` ] }; - await cleanFile(exportPath); + await remove(exportPath); assert.deepEqual(actual, expected); }); - it('should generate a zip file for pages with related documents', async function () { + it.only('should generate a zip file for pages with related documents', async function () { const req = apos.task.getReq(); const page1 = await apos.page.find(req, { title: 'page1' }).toObject(); const { _id: attachmentId } = await apos.attachment.db.findOne({ name: 'test-image' }); @@ -170,7 +170,7 @@ describe('@apostrophecms/import-export', function () { const fileName = path.basename(url); pageTgzPath = path.join(exportsPath, fileName); - const exportPath = await gzip.input(pageTgzPath); + const { exportPath } = await gzip.input(pageTgzPath); const { docs, attachments, attachmentFiles @@ -217,10 +217,10 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await cleanFile(exportPath); + await remove(exportPath); }); - it('should import pieces with related documents from a compressed file', async function() { + it.only('should import pieces with related documents from a compressed file', async function() { const req = apos.task.getReq(); await deletePieces(apos); @@ -271,7 +271,7 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await cleanFile(piecesTgzPath.replace(gzip.allowedExtension, '')); + await remove(piecesTgzPath.replace(gzip.allowedExtension, '')); }); it('should return duplicates pieces when already existing and override them', async function() { @@ -354,7 +354,7 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await cleanFile(piecesTgzPath.replace(gzip.allowedExtension, '')); + await remove(piecesTgzPath.replace(gzip.allowedExtension, '')); }); it('should import page and related documents', async function() { @@ -396,7 +396,7 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await cleanFile(pageTgzPath.replace(gzip.allowedExtension, '')); + await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should return existing duplicated docs during page import and override them', async function() { @@ -485,7 +485,7 @@ describe('@apostrophecms/import-export', function () { assert.deepEqual(actual, expected); - await cleanFile(pageTgzPath.replace(gzip.allowedExtension, '')); + await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should not override attachment if associated document is not imported', async function() { @@ -578,7 +578,7 @@ describe('@apostrophecms/import-export', function () { assert.deepEqual(actual, expected); - await cleanFile(pageTgzPath.replace(gzip.allowedExtension, '')); + await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should preserve lastPublishedAt property on import for existing drafts', async function() { @@ -955,7 +955,7 @@ describe('@apostrophecms/import-export', function () { importExportManager = apos.modules['@apostrophecms/import-export']; importExportManager.removeExportFileFromUploadFs = () => {}; - importExportManager.cleanFile = () => {}; + importExportManager.remove = () => {}; await insertAdminUser(apos); await insertPieces(apos); @@ -1252,7 +1252,7 @@ describe('@apostrophecms/import-export', function () { importExportManager = apos.modules['@apostrophecms/import-export']; importExportManager.removeExportFileFromUploadFs = () => {}; - importExportManager.cleanFile = () => {}; + importExportManager.remove = () => {}; await insertAdminUser(apos); await insertPieces(apos); From ad52ded4f7b9894ea594ebe284c1b8f6dcb1d691 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 22 Apr 2024 16:58:20 +0200 Subject: [PATCH 27/42] wip --- test/index.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/index.js b/test/index.js index 13b77a34..fc03b61f 100644 --- a/test/index.js +++ b/test/index.js @@ -35,7 +35,7 @@ describe('@apostrophecms/import-export', function () { attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); exportsPath = path.join(apos.rootDir, 'public/uploads/exports'); importExportManager = apos.modules['@apostrophecms/import-export']; - importExportManager.removeExportFileFromUploadFs = () => {}; + importExportManager.removeFromUploadFs = () => {}; gzip = importExportManager.formats.gzip; mimeType = gzip.allowedTypes[0]; remove = importExportManager.remove; @@ -79,7 +79,7 @@ describe('@apostrophecms/import-export', function () { assert.deepEqual(actual, expected); }); - it.only('should generate a zip file for pieces with related documents', async function () { + it('should generate a zip file for pieces with related documents', async function () { const req = apos.task.getReq(); const articles = await apos.article.find(req).toArray(); const { _id: attachmentId } = await apos.attachment.db.findOne({ name: 'test-image' }); @@ -150,11 +150,11 @@ describe('@apostrophecms/import-export', function () { attachmentFiles: [ `${attachmentId}-test-image.jpg` ] }; - await remove(exportPath); + // await remove(exportPath); assert.deepEqual(actual, expected); }); - it.only('should generate a zip file for pages with related documents', async function () { + it('should generate a zip file for pages with related documents', async function () { const req = apos.task.getReq(); const page1 = await apos.page.find(req, { title: 'page1' }).toObject(); const { _id: attachmentId } = await apos.attachment.db.findOne({ name: 'test-image' }); @@ -220,7 +220,7 @@ describe('@apostrophecms/import-export', function () { await remove(exportPath); }); - it.only('should import pieces with related documents from a compressed file', async function() { + it('should import pieces with related documents from a compressed file', async function() { const req = apos.task.getReq(); await deletePieces(apos); From 7c055c82c9643453af6e77ecce10ec476b297630 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 24 Apr 2024 15:55:32 +0200 Subject: [PATCH 28/42] remove xlsx format because moved to a separate module --- README.md | 40 ++++++++++++++++++++++++++++++++ lib/formats/index.js | 4 +--- lib/formats/xlsx.js | 55 -------------------------------------------- package.json | 3 +-- 4 files changed, 42 insertions(+), 60 deletions(-) delete mode 100644 lib/formats/xlsx.js diff --git a/README.md b/README.md index 9a0a8331..56e9925b 100644 --- a/README.md +++ b/README.md @@ -271,3 +271,43 @@ module.exports = { } }; ``` + +### Add formats from a separate module + +You might want to scope one or multiple formats in another module for several reasons: + +- The formats rely on a dependency that is not hosted on NPM (which is the case with [@apostrophecms/import-export-xlsx](https://github.com/apostrophecms/import-export-xlsx)) +- You want to fully scope the format in a separate module and repository for an easier maintenance +- ... + +To do so, simply create an apostrophe module that improves `@apostrophecms/import-export` and define the format in the `formats` option of the module. + +Then add the module to the project **package.json** and **app.js**. + +Example with an `import-export-excel` module: + +```js +// index.js +module.exports = { + improve: '@apostrophecms/import-export', + options: { + formats: {, + xls: { + // ... + }, + xlsx: { + label: 'XLSX', + extension: '.xlsx', + allowedExtension: '.xlsx', + allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], + async input(filepath) { + // ... + }, + async output(filepath, { docs }) { + // ... + } + } + } + } +}; +``` diff --git a/lib/formats/index.js b/lib/formats/index.js index 89b1d50a..b916f349 100644 --- a/lib/formats/index.js +++ b/lib/formats/index.js @@ -1,10 +1,8 @@ const gzip = require('./gzip'); const csv = require('./csv'); -const xlsx = require('./xlsx'); // Keep gzip at the top of the list so it's the default format module.exports = { gzip, - csv, - xlsx // TODO: make it a separate module to register the XLSX format, and make formats registerable: import-export-xlsx + csv }; diff --git a/lib/formats/xlsx.js b/lib/formats/xlsx.js deleted file mode 100644 index 9c2d5d9a..00000000 --- a/lib/formats/xlsx.js +++ /dev/null @@ -1,55 +0,0 @@ -const XLSX = require('xlsx'); - -const SHEET_NAME = 'docs'; - -module.exports = { - label: 'XLSX', - extension: '.xlsx', - allowedExtension: '.xlsx', - allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], - async input(filepath) { - const workbook = XLSX.readFile(filepath); - - const sheet = workbook.Sheets[SHEET_NAME]; - const docs = XLSX.utils.sheet_to_json(sheet); - - return { docs: docs.map(parse) }; - }, - async output(filepath, { docs }) { - const workbook = XLSX.utils.book_new(); - const worksheet = XLSX.utils.json_to_sheet( - docs.map(stringify) - ); - - XLSX.utils.book_append_sheet(workbook, worksheet, SHEET_NAME); - XLSX.writeFile(workbook, filepath, { compression: true }); - - console.info(`[xlsx] docs and attachments written to ${filepath}`); - } -}; - -function stringify(doc) { - const object = {}; - for (const key in doc) { - object[key] = typeof doc[key] === 'object' - ? JSON.stringify(doc[key]) - : doc[key]; - } - return object; -} - -function parse(doc) { - const object = {}; - for (const key in doc) { - if (!doc[key]) { - // Avoid setting empty values as empty strings - continue; - } - try { - object[key] = JSON.parse(doc[key]); - } catch { - object[key] = doc[key]; - } - } - return object; -} diff --git a/package.json b/package.json index 71dadc81..c7a202c8 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,6 @@ "csv-stringify": "^6.4.6", "dayjs": "^1.9.8", "lodash": "^4.17.21", - "tar-stream": "^3.1.6", - "xlsx": "https://cdn.sheetjs.com/xlsx-0.20.2/xlsx-0.20.2.tgz" + "tar-stream": "^3.1.6" } } From 74f1b0a6a2194c21c380311f809e3b4772f0d0cb Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 24 Apr 2024 15:59:00 +0200 Subject: [PATCH 29/42] better readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 56e9925b..4a5fbf8d 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,7 @@ module.exports = { }; ``` -**Note**: The `import` and `export` functions should remain agnostic of any apostrophe logic. +**Note**: The `input` and `output` functions should remain agnostic of any apostrophe logic. ```js // lib/formats/index.js @@ -191,7 +191,7 @@ module.exports = { ### For formats with attachment files: -If you want to add a format that includes attachment files such as an archive, you can enable the `includeAttachments` option and utilize extra arguments provided in the `import` and `export` functions. +If you want to add a format that includes attachment files such as an archive, you can enable the `includeAttachments` option and utilize extra arguments provided in the `input` and `output` functions. **Advanced example**: @@ -272,7 +272,7 @@ module.exports = { }; ``` -### Add formats from a separate module +### Add formats via a separate module You might want to scope one or multiple formats in another module for several reasons: From 0f345db1c0872bb3166defb65085ec7d2c027bc8 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 24 Apr 2024 18:53:25 +0200 Subject: [PATCH 30/42] add registerFormats method --- lib/methods/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/methods/index.js b/lib/methods/index.js index 5c7840f4..dbf55b1a 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -4,6 +4,12 @@ const exportMethods = require('./export'); module.exports = self => { return { + registerFormats(formats = {}) { + self.formats = { + ...self.formats, + ...formats + }; + }, // No need to override, the parent method returns `{}`. getBrowserData() { return { From 7fb0a84e33ea95b283045578a7e1c2c914ecaa54 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Wed, 24 Apr 2024 18:59:16 +0200 Subject: [PATCH 31/42] edit doc with the registration --- README.md | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 4a5fbf8d..068b5fc9 100644 --- a/README.md +++ b/README.md @@ -280,34 +280,37 @@ You might want to scope one or multiple formats in another module for several re - You want to fully scope the format in a separate module and repository for an easier maintenance - ... -To do so, simply create an apostrophe module that improves `@apostrophecms/import-export` and define the format in the `formats` option of the module. +To do so, simply create an apostrophe module that improves `@apostrophecms/import-export` and register the formats in the `init` method. -Then add the module to the project **package.json** and **app.js**. Example with an `import-export-excel` module: ```js -// index.js +const formats: { + xls: { + label: 'XLS', + extension: '.xls', + allowedExtension: '.xls', + allowedTypes: [ 'application/vnd.ms-excel' ], + async input(filepath) {}, + async output(filepath, { docs }) {} + }, + xlsx: { + label: 'XLSX', + extension: '.xlsx', + allowedExtension: '.xlsx', + allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], + async input(filepath) {}, + async output(filepath, { docs }) {} + } +}; + module.exports = { improve: '@apostrophecms/import-export', - options: { - formats: {, - xls: { - // ... - }, - xlsx: { - label: 'XLSX', - extension: '.xlsx', - allowedExtension: '.xlsx', - allowedTypes: [ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' ], - async input(filepath) { - // ... - }, - async output(filepath, { docs }) { - // ... - } - } - } + init(self) { + self.registerFormats(formats); } }; ``` + +Then add the module to the project **package.json** and **app.js**. From 2a0a2533be739086c5bebd81f4e979cdbb1f5a98 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 16:34:29 +0200 Subject: [PATCH 32/42] verify registered formats --- lib/methods/import.js | 1 - lib/methods/index.js | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index 656e6ab4..7ea49555 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -33,7 +33,6 @@ module.exports = self => { if (!file) { throw new Error('invalid: no file provided'); } - console.log(file); format = Object .values(self.formats) diff --git a/lib/methods/index.js b/lib/methods/index.js index dbf55b1a..ce27050c 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -5,6 +5,8 @@ const exportMethods = require('./export'); module.exports = self => { return { registerFormats(formats = {}) { + verifyFormats(formats); + self.formats = { ...self.formats, ...formats @@ -66,3 +68,63 @@ module.exports = self => { ...exportMethods(self) }; }; + +function verifyFormats(formats) { + if (typeof formats !== 'object') { + throw new Error('formats must be an object'); + } + + Object + .entries(formats) + .forEach(([ formatName, format ]) => { + const allowedKeys = [ 'label', 'extension', 'allowedExtension', 'allowedTypes', 'includeAttachments', 'input', 'output' ]; + const requiredKeys = [ 'label', 'extension', 'allowedExtension', 'allowedTypes', 'input', 'output' ]; + + const keys = Object.keys(format); + + if (requiredKeys.some(requiredKey => !keys.includes(requiredKey))) { + throw new Error(`${formatName}.label, ${formatName}.extension, ${formatName}.allowedExtension, ${formatName}.allowedTypes, ${formatName}.input and ${formatName}.output are required keys`); + } + keys.forEach(key => { + if (!allowedKeys.includes(key)) { + throw new Error(`format.${key} is not a valid key`); + } + }); + keys.forEach(key => { + if (key === 'label') { + if (typeof format.label !== 'string') { + throw new Error(`${formatName}.label must be a string`); + } + } else if (key === 'extension') { + if (typeof format.extension !== 'string') { + throw new Error(`${formatName}.extension must be a string`); + } + } else if (key === 'allowedExtension') { + if (typeof format.allowedExtension !== 'string') { + throw new Error(`${formatName}.allowedExtension must be a string`); + } + } else if (key === 'allowedTypes') { + if (!Array.isArray(format.allowedTypes)) { + throw new Error(`${formatName}.allowedTypes must be an array`); + } + format.allowedTypes.forEach(allowedType => { + if (typeof allowedType !== 'string') { + throw new Error(`${formatName}.allowedTypes must be an array of strings`); + } + }); + } else if (key === 'includeAttachments') { + if (typeof format.includeAttachments !== 'boolean') { + throw new Error(`${formatName}.includeAttachments must be a boolean`); + } + } else if (key === 'input') { + if (typeof format.input !== 'function') { + throw new Error(`${formatName}.input must be a function`); + } + } else if (key === 'output') { + if (typeof format.output !== 'function') { + throw new Error(`${formatName}.output must be a function`); + } + } + }); + }); +}; From 4ddd45638532695f2a2ac4c40c11cf65b852faec Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 16:36:10 +0200 Subject: [PATCH 33/42] verify registered formats - better --- lib/methods/index.js | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/methods/index.js b/lib/methods/index.js index ce27050c..06f5f05b 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -87,42 +87,42 @@ function verifyFormats(formats) { } keys.forEach(key => { if (!allowedKeys.includes(key)) { - throw new Error(`format.${key} is not a valid key`); + throw new Error(`${formatName}.${key} is not a valid key`); } }); keys.forEach(key => { if (key === 'label') { if (typeof format.label !== 'string') { - throw new Error(`${formatName}.label must be a string`); + throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'extension') { if (typeof format.extension !== 'string') { - throw new Error(`${formatName}.extension must be a string`); + throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'allowedExtension') { if (typeof format.allowedExtension !== 'string') { - throw new Error(`${formatName}.allowedExtension must be a string`); + throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'allowedTypes') { if (!Array.isArray(format.allowedTypes)) { - throw new Error(`${formatName}.allowedTypes must be an array`); + throw new Error(`${formatName}.${key} must be an array`); } format.allowedTypes.forEach(allowedType => { if (typeof allowedType !== 'string') { - throw new Error(`${formatName}.allowedTypes must be an array of strings`); + throw new Error(`${formatName}.${key} must be an array of strings`); } }); } else if (key === 'includeAttachments') { if (typeof format.includeAttachments !== 'boolean') { - throw new Error(`${formatName}.includeAttachments must be a boolean`); + throw new Error(`${formatName}.${key} must be a boolean`); } } else if (key === 'input') { if (typeof format.input !== 'function') { - throw new Error(`${formatName}.input must be a function`); + throw new Error(`${formatName}.${key} must be a function`); } } else if (key === 'output') { if (typeof format.output !== 'function') { - throw new Error(`${formatName}.output must be a function`); + throw new Error(`${formatName}.${key} must be a function`); } } }); From f094c9165238f16ad1513eb4d91cd4f0d0d8f4d2 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 16:37:56 +0200 Subject: [PATCH 34/42] verify registered formats - better --- lib/methods/index.js | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/methods/index.js b/lib/methods/index.js index 06f5f05b..bf72dceb 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -92,36 +92,36 @@ function verifyFormats(formats) { }); keys.forEach(key => { if (key === 'label') { - if (typeof format.label !== 'string') { + if (typeof format[key] !== 'string') { throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'extension') { - if (typeof format.extension !== 'string') { + if (typeof format[key] !== 'string') { throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'allowedExtension') { - if (typeof format.allowedExtension !== 'string') { + if (typeof format[key] !== 'string') { throw new Error(`${formatName}.${key} must be a string`); } } else if (key === 'allowedTypes') { - if (!Array.isArray(format.allowedTypes)) { + if (!Array.isArray(format[key])) { throw new Error(`${formatName}.${key} must be an array`); } - format.allowedTypes.forEach(allowedType => { + format[key].forEach(allowedType => { if (typeof allowedType !== 'string') { throw new Error(`${formatName}.${key} must be an array of strings`); } }); } else if (key === 'includeAttachments') { - if (typeof format.includeAttachments !== 'boolean') { + if (typeof format[key] !== 'boolean') { throw new Error(`${formatName}.${key} must be a boolean`); } } else if (key === 'input') { - if (typeof format.input !== 'function') { + if (typeof format[key] !== 'function') { throw new Error(`${formatName}.${key} must be a function`); } } else if (key === 'output') { - if (typeof format.output !== 'function') { + if (typeof format[key] !== 'function') { throw new Error(`${formatName}.${key} must be a function`); } } From bb4594a4569862eb9c83d27f8557794da6d0c889 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 16:38:45 +0200 Subject: [PATCH 35/42] verify registered formats - better --- lib/methods/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/methods/index.js b/lib/methods/index.js index bf72dceb..8d0584ee 100644 --- a/lib/methods/index.js +++ b/lib/methods/index.js @@ -77,8 +77,8 @@ function verifyFormats(formats) { Object .entries(formats) .forEach(([ formatName, format ]) => { - const allowedKeys = [ 'label', 'extension', 'allowedExtension', 'allowedTypes', 'includeAttachments', 'input', 'output' ]; const requiredKeys = [ 'label', 'extension', 'allowedExtension', 'allowedTypes', 'input', 'output' ]; + const allowedKeys = [ ...requiredKeys, 'includeAttachments' ]; const keys = Object.keys(format); From 5dfa8034496bba30f31357f2ff32f8ce1933770e Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 18:49:06 +0200 Subject: [PATCH 36/42] adapt tests - the end --- lib/methods/import.js | 9 +- test/index.js | 303 ++++++++++++++++++++++++++++-------------- 2 files changed, 207 insertions(+), 105 deletions(-) diff --git a/lib/methods/import.js b/lib/methods/import.js index 7ea49555..ea1463ee 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -20,14 +20,15 @@ module.exports = self => { if (!formatLabel) { throw self.apos.error('invalid: no `formatLabel` provided'); } - if (!exportPath) { - throw self.apos.error('invalid: no `exportPath` provided'); - } format = Object .values(self.formats) .find(format => format.label === formatLabel); + if (!exportPath) { + throw self.apos.error('invalid: no `exportPath` provided'); + } + ({ docs, attachmentsInfo = [] } = await format.input(exportPath)); } else { if (!file) { @@ -48,7 +49,7 @@ module.exports = self => { } } catch (error) { await self.apos.notify(req, 'aposImportExport:importFileError', { - interpolate: { format: format.label }, + interpolate: { format: format?.label }, dismiss: true, icon: 'alert-circle-icon', type: 'danger' diff --git a/test/index.js b/test/index.js index fc03b61f..06a8dd66 100644 --- a/test/index.js +++ b/test/index.js @@ -10,21 +10,16 @@ const FormData = require('form-data'); describe('@apostrophecms/import-export', function () { let apos; let importExportManager; + let tempPath; let attachmentPath; let exportsPath; let gzip; let mimeType; let piecesTgzPath; let pageTgzPath; - let remove; this.timeout(60000); - after(async function() { - await cleanData([ attachmentPath, exportsPath ]); - await t.destroy(apos); - }); - before(async function() { apos = await t.create({ root: module, @@ -32,20 +27,33 @@ describe('@apostrophecms/import-export', function () { modules: getAppConfig() }); + tempPath = path.join(apos.rootDir, 'data/temp/uploadfs'); attachmentPath = path.join(apos.rootDir, 'public/uploads/attachments'); exportsPath = path.join(apos.rootDir, 'public/uploads/exports'); importExportManager = apos.modules['@apostrophecms/import-export']; importExportManager.removeFromUploadFs = () => {}; + importExportManager.remove = () => {}; gzip = importExportManager.formats.gzip; mimeType = gzip.allowedTypes[0]; - remove = importExportManager.remove; - importExportManager.remove = () => {}; await insertAdminUser(apos); - await insertPieces(apos); }); - it.only('should generate a zip file for pieces without related documents', async function () { + after(async function() { + await deletePiecesAndPages(apos); + await deleteAttachments(apos, attachmentPath); + await t.destroy(apos); + await cleanData([ tempPath, exportsPath, attachmentPath ]); + }); + + beforeEach(async function() { + await cleanData([ tempPath, exportsPath, attachmentPath ]); + await deletePiecesAndPages(apos); + await deleteAttachments(apos, attachmentPath); + await insertPiecesAndPages(apos); + }); + + it('should generate a zip file for pieces without related documents', async function () { const req = apos.task.getReq(); const articles = await apos.article.find(req).toArray(); const manager = apos.article; @@ -75,7 +83,6 @@ describe('@apostrophecms/import-export', function () { attachmentFiles: [] }; - await remove(exportPath); assert.deepEqual(actual, expected); }); @@ -150,7 +157,6 @@ describe('@apostrophecms/import-export', function () { attachmentFiles: [ `${attachmentId}-test-image.jpg` ] }; - // await remove(exportPath); assert.deepEqual(actual, expected); }); @@ -217,14 +223,26 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await remove(exportPath); }); it('should import pieces with related documents from a compressed file', async function() { const req = apos.task.getReq(); + const articles = await apos.article.find(req).toArray(); + const manager = apos.article; + + req.body = { + _ids: articles.map(({ _id }) => _id), + extension: 'gzip', + relatedTypes: [ '@apostrophecms/image', 'topic' ], + type: req.t(manager.options.pluralLabel) + }; + + const { url } = await importExportManager.export(req, manager); + const fileName = path.basename(url); - await deletePieces(apos); - await deletePage(apos); + piecesTgzPath = path.join(exportsPath, fileName); + + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); req.body = {}; @@ -271,11 +289,24 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await remove(piecesTgzPath.replace(gzip.allowedExtension, '')); }); it('should return duplicates pieces when already existing and override them', async function() { const req = apos.task.getReq(); + const articles = await apos.article.find(req).toArray(); + const manager = apos.article; + + req.body = { + _ids: articles.map(({ _id }) => _id), + extension: 'gzip', + relatedTypes: [ '@apostrophecms/image', 'topic' ], + type: req.t(manager.options.pluralLabel) + }; + + const { url } = await importExportManager.export(req, manager); + const fileName = path.basename(url); + + piecesTgzPath = path.join(exportsPath, fileName); req.body = {}; req.files = { @@ -290,7 +321,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel } = await importExportManager.import(req); // We update the title of every targetted docs to be sure the update really occurs @@ -312,7 +344,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel }; await importExportManager.overrideDuplicates(req); @@ -339,10 +372,18 @@ describe('@apostrophecms/import-export', function () { const expected = { docTitles: [ - 'article2', 'article1', - 'article2', 'article1', - 'topic1', 'topic2', - 'topic1', 'topic2' + 'image1', + 'image1', + 'article1', + 'article2', + 'article1', + 'article2', + 'new title', + 'topic2', + 'topic1', + 'new title', + 'topic2', + 'topic1' ], attachmentNames: [ 'test-image' ], attachmentFileNames: new Array(apos.attachment.imageSizes.length + 1) @@ -354,14 +395,25 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await remove(piecesTgzPath.replace(gzip.allowedExtension, '')); }); it('should import page and related documents', async function() { const req = apos.task.getReq(); + const page1 = await apos.page.find(req, { title: 'page1' }).toObject(); + + req.body = { + _ids: [ page1._id ], + extension: 'gzip', + relatedTypes: [ '@apostrophecms/image', 'article' ], + type: page1.type + }; + + const { url } = await importExportManager.export(req, apos.page); + const fileName = path.basename(url); + + pageTgzPath = path.join(exportsPath, fileName); - await deletePieces(apos); - await deletePage(apos); + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); req.body = {}; @@ -396,11 +448,23 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should return existing duplicated docs during page import and override them', async function() { const req = apos.task.getReq(); + const page1 = await apos.page.find(req, { title: 'page1' }).toObject(); + + req.body = { + _ids: [ page1._id ], + extension: 'gzip', + relatedTypes: [ '@apostrophecms/image', 'article' ], + type: page1.type + }; + + const { url } = await importExportManager.export(req, apos.page); + const fileName = path.basename(url); + + pageTgzPath = path.join(exportsPath, fileName); req.body = {}; req.files = { @@ -415,7 +479,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel } = await importExportManager.import(req); // We update the title of every targetted docs to be sure the update really occurs @@ -440,7 +505,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel }; await importExportManager.overrideDuplicates(req); @@ -469,7 +535,9 @@ describe('@apostrophecms/import-export', function () { docTitles: [ 'image1', 'image1', + 'new title', 'article2', + 'new title', 'article2', 'page1', 'page1' @@ -484,12 +552,23 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - - await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should not override attachment if associated document is not imported', async function() { const req = apos.task.getReq(); + const page1 = await apos.page.find(req, { title: 'page1' }).toObject(); + + req.body = { + _ids: [ page1._id ], + extension: 'gzip', + relatedTypes: [ '@apostrophecms/image', 'article' ], + type: page1.type + }; + + const { url } = await importExportManager.export(req, apos.page); + const fileName = path.basename(url); + + pageTgzPath = path.join(exportsPath, fileName); req.body = {}; req.files = { @@ -504,7 +583,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel } = await importExportManager.import(req); // We update the title of every targetted docs to be sure the update really occurs @@ -531,7 +611,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel }; await importExportManager.overrideDuplicates(req); @@ -559,9 +640,11 @@ describe('@apostrophecms/import-export', function () { const expected = { docTitles: [ + 'new title', 'new title', 'new title', 'article2', + 'new title', 'article2', 'page1', 'page1' @@ -577,8 +660,6 @@ describe('@apostrophecms/import-export', function () { }; assert.deepEqual(actual, expected); - - await remove(pageTgzPath.replace(gzip.allowedExtension, '')); }); it('should preserve lastPublishedAt property on import for existing drafts', async function() { @@ -633,7 +714,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel } = await importExportManager.import(req); req.body = { @@ -641,7 +723,8 @@ describe('@apostrophecms/import-export', function () { importedAttachments, exportPathId, jobId, - notificationId + notificationId, + formatLabel }; await importExportManager.overrideDuplicates(req); @@ -747,51 +830,54 @@ describe('@apostrophecms/import-export', function () { describe('#import - overriding locales integration tests', function() { let req; let notify; - let getFilesData; - let readExportFile; + let input; let rewriteDocsWithCurrentLocale; let insertDocs; this.beforeEach(async function() { req = apos.task.getReq({ locale: 'en', - body: {} + body: {}, + files: { + file: { + path: '/some/path/to/file', + type: mimeType + } + } }); notify = apos.notify; - getFilesData = apos.modules['@apostrophecms/import-export'].getFilesData; - readExportFile = apos.modules['@apostrophecms/import-export'].readExportFile; + input = gzip.input; rewriteDocsWithCurrentLocale = apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale; insertDocs = apos.modules['@apostrophecms/import-export'].insertDocs; - await deletePieces(apos); - await deletePage(apos); + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); }); this.afterEach(function() { apos.notify = notify; - apos.modules['@apostrophecms/import-export'].getFilesData = getFilesData; - apos.modules['@apostrophecms/import-export'].readExportFile = readExportFile; + gzip.input = input; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = rewriteDocsWithCurrentLocale; apos.modules['@apostrophecms/import-export'].insertDocs = insertDocs; }); it('should import pieces with related documents from the extracted export path when provided', async function() { - const expectedPath = '/custom/extracted-export-path'; // Since we are mocking this and not really uploading a file, we have to // manually call setExportPathId to establish a mapping to a safe // unique identifier to share with the "browser" + const expectedPath = '/custom/extracted-export-path'; await importExportManager.setExportPathId(expectedPath); - const req = apos.task.getReq({ + + req = apos.task.getReq({ + locale: 'en', body: { - exportPathId: await importExportManager.getExportPathId(expectedPath) + exportPathId: await importExportManager.getExportPathId(expectedPath), + formatLabel: 'gzip', + overrideLocale: true } }); - apos.modules['@apostrophecms/import-export'].readExportFile = async () => { - throw new Error('should not have been called'); - }; - apos.modules['@apostrophecms/import-export'].getFilesData = async exportPath => { + gzip.input = async exportPath => { assert.equal(exportPath, expectedPath); return { @@ -813,7 +899,7 @@ describe('@apostrophecms/import-export', function () { describe('when the site has only one locale', function() { it('should not rewrite the docs locale nor ask about it when the locale is not different', async function() { - apos.modules['@apostrophecms/import-export'].readExportFile = async req => { + gzip.input = async () => { return { docs: [ { @@ -828,7 +914,7 @@ describe('@apostrophecms/import-export', function () { }; }; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = (req, docs) => { - throw new Error('should not have been called'); + throw new Error('rewriteDocsWithCurrentLocale should not have been called'); }; apos.modules['@apostrophecms/import-export'].insertDocs = async (req, docs) => { assert.deepEqual(docs, [ @@ -849,7 +935,7 @@ describe('@apostrophecms/import-export', function () { }; apos.notify = async (req, message, options) => { if (options?.event?.name === 'import-export-import-locale-differs') { - throw new Error('should not have been called with event "import-locale-differ"'); + throw new Error('notify should not have been called with event "import-locale-differ"'); } return {}; }; @@ -858,7 +944,7 @@ describe('@apostrophecms/import-export', function () { }); it('should rewrite the docs locale without asking about it when the locale is different', async function() { - apos.modules['@apostrophecms/import-export'].readExportFile = async req => { + gzip.input = async () => { return { docs: [ { @@ -904,7 +990,7 @@ describe('@apostrophecms/import-export', function () { }; apos.notify = async (req, message, options) => { if (options?.event?.name === 'import-export-import-locale-differs') { - throw new Error('should not have been called with event "import-locale-differ"'); + throw new Error('notify should not have been called with event "import-locale-differ"'); } return {}; }; @@ -919,8 +1005,7 @@ describe('@apostrophecms/import-export', function () { let req; let notify; - let getFilesData; - let readExportFile; + let input; let rewriteDocsWithCurrentLocale; let insertDocs; @@ -958,29 +1043,32 @@ describe('@apostrophecms/import-export', function () { importExportManager.remove = () => {}; await insertAdminUser(apos); - await insertPieces(apos); + await insertPiecesAndPages(apos); }); this.beforeEach(async function() { req = apos.task.getReq({ locale: 'en', - body: {} + body: {}, + files: { + file: { + path: '/some/path/to/file', + type: mimeType + } + } }); notify = apos.notify; - getFilesData = apos.modules['@apostrophecms/import-export'].getFilesData; - readExportFile = apos.modules['@apostrophecms/import-export'].readExportFile; + input = gzip.input; rewriteDocsWithCurrentLocale = apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale; insertDocs = apos.modules['@apostrophecms/import-export'].insertDocs; - await deletePieces(apos); - await deletePage(apos); + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); }); this.afterEach(function() { apos.notify = notify; - apos.modules['@apostrophecms/import-export'].getFilesData = getFilesData; - apos.modules['@apostrophecms/import-export'].readExportFile = readExportFile; + gzip.input = input; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = rewriteDocsWithCurrentLocale; apos.modules['@apostrophecms/import-export'].insertDocs = insertDocs; }); @@ -988,10 +1076,16 @@ describe('@apostrophecms/import-export', function () { it('should not rewrite the docs locale nor ask about it when the locale is not different', async function() { const req = apos.task.getReq({ locale: 'fr', - body: {} + body: {}, + files: { + file: { + path: '/some/path/to/file', + type: mimeType + } + } }); - apos.modules['@apostrophecms/import-export'].readExportFile = async req => { + gzip.input = async req => { return { docs: [ { @@ -1006,7 +1100,7 @@ describe('@apostrophecms/import-export', function () { }; }; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = () => { - throw new Error('should not have been called'); + throw new Error('rewriteDocsWithCurrentLocale should not have been called'); }; apos.modules['@apostrophecms/import-export'].insertDocs = async (req, docs) => { assert.deepEqual(docs, [ @@ -1027,7 +1121,7 @@ describe('@apostrophecms/import-export', function () { }; apos.notify = async (req, message, options) => { if (options?.event?.name === 'import-export-import-locale-differs') { - throw new Error('should not have been called with event "import-locale-differ"'); + throw new Error('notify should not have been called with event "import-locale-differ"'); } return {}; }; @@ -1036,7 +1130,7 @@ describe('@apostrophecms/import-export', function () { }); it('should not rewrite the docs locales nor insert them but ask about it when the locale is different', async function() { - apos.modules['@apostrophecms/import-export'].readExportFile = async req => { + gzip.input = async req => { return { docs: [ { @@ -1052,10 +1146,10 @@ describe('@apostrophecms/import-export', function () { }; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = () => { - throw new Error('should not have been called'); + throw new Error('rewriteDocsWithCurrentLocale should not have been called'); }; apos.modules['@apostrophecms/import-export'].insertDocs = async (req, docs) => { - throw new Error('should not have been called'); + throw new Error('insertDocs should not have been called'); }; apos.notify = async (req, message, options) => { assert.equal(options.event.name, 'import-export-import-locale-differs'); @@ -1065,14 +1159,22 @@ describe('@apostrophecms/import-export', function () { }); it('should rewrite the docs locale when the locale is different and the `overrideLocale` param is provided', async function() { + // Since we are mocking this and not really uploading a file, we have to + // manually call setExportPathId to establish a mapping to a safe + // unique identifier to share with the "browser" + const expectedPath = '/custom/extracted-export-path'; + await importExportManager.setExportPathId(expectedPath); + const req = apos.task.getReq({ locale: 'en', body: { + exportPathId: await importExportManager.getExportPathId(expectedPath), + formatLabel: 'gzip', overrideLocale: true } }); - apos.modules['@apostrophecms/import-export'].readExportFile = async req => { + gzip.input = async req => { return { docs: [ { @@ -1119,7 +1221,7 @@ describe('@apostrophecms/import-export', function () { }; apos.notify = async (req, message, options) => { if (options?.event?.name === 'import-export-import-locale-differs') { - throw new Error('should not have been called with event "import-locale-differ"'); + throw new Error('notify should not have been called with event "import-locale-differ"'); } return {}; }; @@ -1131,36 +1233,37 @@ describe('@apostrophecms/import-export', function () { describe('#overrideDuplicates - overriding locales integration tests', function() { let req; - let getFilesData; + let input; let rewriteDocsWithCurrentLocale; let jobManager; this.beforeEach(async function() { req = apos.task.getReq({ locale: 'en', - body: {} + body: { + formatLabel: 'gzip' + } }); jobManager = apos.modules['@apostrophecms/job']; - getFilesData = apos.modules['@apostrophecms/import-export'].getFilesData; + input = gzip.input; rewriteDocsWithCurrentLocale = apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale; jobManager.success = () => {}; jobManager.failure = () => {}; - await deletePieces(apos); - await deletePage(apos); + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); }); this.afterEach(function() { + gzip.input = input; apos.modules['@apostrophecms/job'].jobManager = jobManager; - apos.modules['@apostrophecms/import-export'].getFilesData = getFilesData; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = rewriteDocsWithCurrentLocale; }); describe('when the site has only one locale', function() { it('should not rewrite the docs locale when the locale is not different', async function() { - apos.modules['@apostrophecms/import-export'].getFilesData = async exportPath => { + gzip.input = async exportPath => { return { docs: [ { @@ -1175,14 +1278,14 @@ describe('@apostrophecms/import-export', function () { }; }; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = (req, docs) => { - throw new Error('should not have been called'); + throw new Error('rewriteDocsWithCurrentLocale should not have been called'); }; await importExportManager.overrideDuplicates(req); }); it('should rewrite the docs locale when the locale is different', async function() { - apos.modules['@apostrophecms/import-export'].getFilesData = async exportPath => { + gzip.input = async exportPath => { return { docs: [ { @@ -1218,7 +1321,7 @@ describe('@apostrophecms/import-export', function () { let apos; let importExportManager; - let getFilesData; + let input; let rewriteDocsWithCurrentLocale; after(async function() { @@ -1255,33 +1358,34 @@ describe('@apostrophecms/import-export', function () { importExportManager.remove = () => {}; await insertAdminUser(apos); - await insertPieces(apos); + await insertPiecesAndPages(apos); }); this.beforeEach(async function() { req = apos.task.getReq({ locale: 'en', - body: {} + body: { + formatLabel: 'gzip' + } }); - getFilesData = apos.modules['@apostrophecms/import-export'].getFilesData; + input = gzip.input; rewriteDocsWithCurrentLocale = apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale; jobManager = apos.modules['@apostrophecms/job']; jobManager.success = () => {}; jobManager.failure = () => {}; - await deletePieces(apos); - await deletePage(apos); + await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); }); this.afterEach(function() { - apos.modules['@apostrophecms/import-export'].getFilesData = getFilesData; + gzip.input = input; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = rewriteDocsWithCurrentLocale; }); it('should not rewrite the docs locale when the locale is not different', async function() { - apos.modules['@apostrophecms/import-export'].getFilesData = async exportPath => { + gzip.input = async exportPath => { return { docs: [ { @@ -1296,7 +1400,7 @@ describe('@apostrophecms/import-export', function () { }; }; apos.modules['@apostrophecms/import-export'].rewriteDocsWithCurrentLocale = (req, docs) => { - throw new Error('should not have been called'); + throw new Error('rewriteDocsWithCurrentLocale should not have been called'); }; await importExportManager.overrideDuplicates(req); @@ -1306,11 +1410,12 @@ describe('@apostrophecms/import-export', function () { const req = apos.task.getReq({ locale: 'en', body: { + formatLabel: 'gzip', overrideLocale: true } }); - apos.modules['@apostrophecms/import-export'].getFilesData = async exportPath => { + gzip.input = async exportPath => { return { docs: [ { @@ -1386,20 +1491,16 @@ async function cleanData(paths) { } } -async function deletePieces(apos) { +async function deletePiecesAndPages(apos) { await apos.doc.db.deleteMany({ type: /default-page|article|topic|@apostrophecms\/image/ }); } -async function deletePage(apos) { - await apos.doc.db.deleteMany({ title: 'page1' }); -} - async function deleteAttachments(apos, attachmentPath) { await apos.attachment.db.deleteMany({}); await cleanData([ attachmentPath ]); } -async function insertPieces(apos) { +async function insertPiecesAndPages(apos) { const req = apos.task.getReq(); const formData = new FormData(); From bf3c24c8ef6dc01bd4215ad54f966279336460fd Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:07:17 +0200 Subject: [PATCH 37/42] clean logs and todos --- lib/formats/csv.js | 1 - lib/formats/gzip.js | 2 -- lib/methods/import.js | 1 - 3 files changed, 4 deletions(-) diff --git a/lib/formats/csv.js b/lib/formats/csv.js index 37ef417a..caa67797 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -56,7 +56,6 @@ module.exports = { return value.toISOString(); }, boolean(value) { - // TODO: check return value ? 'true' : 'false'; } } diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index 7517db12..250add83 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -16,14 +16,12 @@ module.exports = { ], includeAttachments: true, async input(filepath) { - console.log('🚀 ~ input ~ filepath:', filepath); let exportPath = filepath; // If the given path is actually the archive, we first need to extract it. // Then we no longer need the archive file, so we remove it. if (filepath.endsWith(this.allowedExtension)) { exportPath = filepath.replace(this.allowedExtension, ''); - console.log('🚀 ~ input ~ exportPath:', exportPath); console.info(`[gzip] extracting ${filepath} into ${exportPath}`); await extract(filepath, exportPath); diff --git a/lib/methods/import.js b/lib/methods/import.js index ea1463ee..71f9033d 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -232,7 +232,6 @@ module.exports = self => { return locale; }, - // TODO: duplicatedDocs are not detected when importing as published only async insertDocs(req, docs, reporting) { const duplicatedDocs = []; const duplicatedIds = []; From a9de59005054cc68ad0267f07a466cb35958aff4 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:13:46 +0200 Subject: [PATCH 38/42] changelog --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6afff5d4..5506ea7b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## UNRELEASED + +### Adds + +* Add CSV format. + +## Breaking changes + +* The signature of the `output` function from the gzip format has changed. It no longer takes the `apos` instance and now requires a `processAttachments` callback. +* `import` and `overrideDuplicates` functions now require `formatLabel` to be passed in `req`. + ## 1.4.1 (2024-03-20) ### Changes From c6de7edbb3bc107e6f2fafad750ae66c77dcbb00 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:24:05 +0200 Subject: [PATCH 39/42] clean data after each test --- test/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/index.js b/test/index.js index 06a8dd66..4baea464 100644 --- a/test/index.js +++ b/test/index.js @@ -40,19 +40,19 @@ describe('@apostrophecms/import-export', function () { }); after(async function() { - await deletePiecesAndPages(apos); - await deleteAttachments(apos, attachmentPath); await t.destroy(apos); - await cleanData([ tempPath, exportsPath, attachmentPath ]); }); beforeEach(async function() { - await cleanData([ tempPath, exportsPath, attachmentPath ]); await deletePiecesAndPages(apos); await deleteAttachments(apos, attachmentPath); await insertPiecesAndPages(apos); }); + afterEach(async function() { + await cleanData([ tempPath, exportsPath, attachmentPath ]); + }); + it('should generate a zip file for pieces without related documents', async function () { const req = apos.task.getReq(); const articles = await apos.article.find(req).toArray(); From 7d791cdb0f10eea70acd226462d85bb924534e6f Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:29:05 +0200 Subject: [PATCH 40/42] changelog major version warning --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d3338fc..bd940ee6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,8 @@ ### Breaking changes +**⚠️ The major version should be incremented: `2.0.0`. Please remove this line before releasing the module.** + * The signature of the `output` function from the gzip format has changed. It no longer takes the `apos` instance and now requires a `processAttachments` callback. * `import` and `overrideDuplicates` functions now require `formatLabel` to be passed in `req`. From e6007ccf9fbd8078005708960f6c8d288bfe1d16 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:31:24 +0200 Subject: [PATCH 41/42] remove try/catch when cleaning data in tests --- test/index.js | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/index.js b/test/index.js index 4baea464..ce3318a8 100644 --- a/test/index.js +++ b/test/index.js @@ -1479,15 +1479,11 @@ async function getExtractedFiles(extractPath) { } async function cleanData(paths) { - try { - for (const filePath of paths) { - const files = await fs.readdir(filePath); - for (const name of files) { - await fs.rm(path.join(filePath, name), { recursive: true }); - } + for (const filePath of paths) { + const files = await fs.readdir(filePath); + for (const name of files) { + await fs.rm(path.join(filePath, name), { recursive: true }); } - } catch (err) { - assert(!err); } } From 08f04810d66682b2d193e9ec449d9f566d9c9b21 Mon Sep 17 00:00:00 2001 From: Etienne Laurent Date: Mon, 29 Apr 2024 19:35:54 +0200 Subject: [PATCH 42/42] remove stuff after each test to fix ci, maybe?? --- test/index.js | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/test/index.js b/test/index.js index ce3318a8..315b24f6 100644 --- a/test/index.js +++ b/test/index.js @@ -44,12 +44,12 @@ describe('@apostrophecms/import-export', function () { }); beforeEach(async function() { - await deletePiecesAndPages(apos); - await deleteAttachments(apos, attachmentPath); await insertPiecesAndPages(apos); }); afterEach(async function() { + await deletePiecesAndPages(apos); + await deleteAttachments(apos, attachmentPath); await cleanData([ tempPath, exportsPath, attachmentPath ]); }); @@ -1479,11 +1479,15 @@ async function getExtractedFiles(extractPath) { } async function cleanData(paths) { - for (const filePath of paths) { - const files = await fs.readdir(filePath); - for (const name of files) { - await fs.rm(path.join(filePath, name), { recursive: true }); + try { + for (const filePath of paths) { + const files = await fs.readdir(filePath); + for (const name of files) { + await fs.rm(path.join(filePath, name), { recursive: true }); + } } + } catch (err) { + assert(!err); } }