diff --git a/lib/formats/csv.js b/lib/formats/csv.js index e6e6f948..1bf19e43 100644 --- a/lib/formats/csv.js +++ b/lib/formats/csv.js @@ -11,7 +11,7 @@ module.exports = { // TODO: remove these? because it really messes with the duplicated docs detection system // importVersions: [ 'draft' ], // exportVersions: [ 'published' ], - async import(exportPath) { + async import(exportPath, { docIds } = {}) { const reader = fs.createReadStream(exportPath); const parser = reader .pipe( @@ -48,16 +48,24 @@ module.exports = { reader.on('error', reject); parser.on('error', reject); parser.on('end', () => { + console.log('IMPORT RETURNS:'); + console.dir({ + docs: !docIds + ? docs + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + exportPath + }, { depth: 9 }); resolve({ docs: !docIds ? docs - : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)) + : docs.filter(({ aposDocId }) => docIds.includes(aposDocId)), + exportPath }); }); }); }, - async export(filepath, { docs }) { - const writer = fs.createWriteStream(filepath); + async export(exportPath, { docs }) { + const writer = fs.createWriteStream(exportPath); const stringifier = stringify({ header: true, columns: getColumnsNames(docs), diff --git a/lib/formats/gzip.js b/lib/formats/gzip.js index d1dcb937..260ad445 100644 --- a/lib/formats/gzip.js +++ b/lib/formats/gzip.js @@ -17,42 +17,59 @@ module.exports = { includeAttachments: true, // importVersions: [ 'draft', 'published' ], // exportVersions: [ 'draft', 'published' ], - async import(filepath) { - const extractPath = filepath.replace(this.allowedExtension, ''); + async import(exportPath, { docIds } = {}) { + console.log('🚀 ~ import ~ exportPath:', exportPath); + console.log('🚀 ~ import ~ docIds:', docIds); + console.log('this.allowedExtension', this.allowedExtension); + // If the given path is actually the archive, we first need to extract it. + // Then we no longer need the archive file, so we remove it. + if (exportPath.endsWith(this.allowedExtension)) { + const filepath = exportPath; + exportPath = exportPath.replace(this.allowedExtension, ''); + await extract(filepath, exportPath); + await remove(filepath); - if (!fs.existsSync(extractPath)) { - await fsp.mkdir(extractPath); } + console.log('🚀 ~ import ~ exportPath:', exportPath); - const readStream = fs.createReadStream(filepath); - const gunzip = zlib.createGunzip(); - const extract = tar.extract(); - - readStream - .pipe(gunzip) - .pipe(extract); - - return new Promise((resolve, reject) => { - readStream.on('error', reject); - gunzip.on('error', reject); - extract.on('error', reject); - - extract.on('entry', (header, stream, next) => { - if (header.type === 'directory') { - fsp - .mkdir(`${extractPath}/${header.name}`) - .then(next) - .catch(reject); - } else { - stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`)); - stream.on('end', next); + const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); + const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); + console.log('IMPORT RETURNS:'); + console.dir({ + docs: !docIds + ? EJSON.parse(docs) + : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join( + exportPath, + 'attachments', + `${attachment._id}-${attachment.name}.${attachment.extension}` + ) } - }); + })), + exportPath + }, { depth: 9 }); - extract.on('finish', () => { - resolve(extractPath); - }); - }); + return { + docs: !docIds + ? EJSON.parse(docs) + : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), + attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ + attachment, + file: { + name: `${attachment.name}.${attachment.extension}`, + path: path.join( + exportPath, + 'attachments', + `${attachment._id}-${attachment.name}.${attachment.extension}` + ) + } + })), + exportPath + }; }, async export( filepath, @@ -117,30 +134,54 @@ module.exports = { }) .catch(reject); }); - }, - async read(exportPath, docIds) { - const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json')); - const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json')); - - return { - docs: !docIds - ? EJSON.parse(docs) - : EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)), - attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({ - attachment, - file: { - name: `${attachment.name}.${attachment.extension}`, - path: path.join( - exportPath, - 'attachments', - `${attachment._id}-${attachment.name}.${attachment.extension}` - ) - } - })) - }; } }; +async function extract(filepath, exportPath) { + console.log('🚀 ~ extract ~ filepath:', filepath); + console.log('🚀 ~ extract ~ exportPath:', exportPath); + + if (!fs.existsSync(exportPath)) { + await fsp.mkdir(exportPath); + } + + const readStream = fs.createReadStream(filepath); + const gunzip = zlib.createGunzip(); + const extract = tar.extract(); + + readStream + .pipe(gunzip) + .pipe(extract); + + return new Promise((resolve, reject) => { + readStream.on('error', reject); + gunzip.on('error', reject); + extract.on('error', reject); + + extract.on('entry', (header, stream, next) => { + if (header.type === 'directory') { + fsp + .mkdir(path.join(exportPath, header.name)) + .then(next) + .catch(reject); + } else { + stream.pipe(fs.WriteStream(path.join(exportPath, header.name))); + stream.on('end', next); + } + }); + extract.on('finish', resolve); + }); +} + +async function remove(filepath) { + console.log('🚀 ~ remove ~ filepath:', filepath); + try { + await fsp.unlink(filepath); + } catch (error) { + self.apos.util.error(error); + } +} + function addTarEntry(pack, options, data = null) { return new Promise((resolve, reject) => { pack.entry(options, data, error => { diff --git a/lib/methods/import.js b/lib/methods/import.js index e84bba4d..ec648ecc 100644 --- a/lib/methods/import.js +++ b/lib/methods/import.js @@ -26,19 +26,23 @@ module.exports = self => { let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); let docs; - let attachmentsInfo = []; + let attachmentsInfo; try { - if (!exportPath) { - exportPath = await format.input(file.path); + if (exportPath) { + ({ + docs = [], + attachmentsInfo = [] + } = await format.import(exportPath)); + } else { + ({ + docs = [], + attachmentsInfo = [], + exportPath + } = await format.import(file.path)); + await self.setExportPathId(exportPath); } - const filesData = await format.getFilesData(exportPath); - - docs = filesData.docs; - attachmentsInfo = filesData.attachmentsInfo; - - await self.cleanFile(file.path); } catch (error) { await self.apos.notify(req, 'aposImportExport:importFileError', { interpolate: { format: format.label }, @@ -85,9 +89,7 @@ module.exports = self => { self.rewriteDocsWithCurrentLocale(req, docs); } - const total = format.includeAttachments - ? docs.length + attachmentsInfo.length - : docs.length; + const total = docs.length + attachmentsInfo.length; const { reporting, jobId, notificationId @@ -97,16 +99,12 @@ module.exports = self => { duplicatedDocs, duplicatedIds, failedIds } = await self.insertDocs(req, docs, reporting); - let importedAttachments = []; - - if (format.includeAttachments) { - importedAttachments = await self.insertAttachments(req, { - attachmentsInfo, - reporting, - duplicatedIds, - docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) - }); - } + const importedAttachments = await self.insertAttachments(req, { + attachmentsInfo, + reporting, + duplicatedIds, + docIds: new Set(docs.map(({ aposDocId }) => aposDocId)) + }); if (!duplicatedDocs.length) { await reporting.end(); @@ -131,7 +129,7 @@ module.exports = self => { self.apos.util.error(error); }); - await self.cleanFile(exportPath); + await self.remove(exportPath); return; } @@ -417,25 +415,6 @@ module.exports = self => { } }, - async cleanFile(path) { - try { - const stat = await fsp.lstat(path); - if (stat.isDirectory()) { - await fsp.rm(path, { - recursive: true, - force: true - }); - } else { - await fsp.unlink(path); - } - } catch (err) { - console.trace(); - self.apos.util.error( - `Error while trying to remove the file or folder: ${path}. You might want to remove it yourself.` - ); - } - }, - async overrideDuplicates(req) { const overrideLocale = self.apos.launder.boolean(req.body.overrideLocale); const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); @@ -458,7 +437,7 @@ module.exports = self => { throw self.apos.error(`invalid format "${formatLabel}"`); } - const { docs, attachmentsInfo } = await format.getFilesData(exportPath, docIds); + const { docs, attachmentsInfo } = await format.import(exportPath, docIds); const differentDocsLocale = self.getFirstDifferentLocale(req, docs); const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1; @@ -521,6 +500,21 @@ module.exports = self => { } }, + async setExportPathId(path) { + const id = self.apos.util.generateId(); + await self.apos.cache.set('exportPaths', id, path, 86400); + await self.apos.cache.set('exportPathIds', path, id, 86400); + return id; + }, + + async getExportPathById(id) { + return self.apos.cache.get('exportPaths', id); + }, + + async getExportPathId(path) { + return self.apos.cache.get('exportPathIds', path); + }, + async cleanExport(req) { const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId)); if (!exportPath) { @@ -538,23 +532,34 @@ module.exports = self => { self.apos.notification.dismiss(req, notificationId, 2000).catch(self.apos.util.error); } - await self.cleanFile(exportPath); - }, - - async setExportPathId(path) { - const id = self.apos.util.generateId(); - await self.apos.cache.set('exportPaths', id, path, 86400); - await self.apos.cache.set('exportPathIds', path, id, 86400); - return id; - }, - - async getExportPathById(id) { - return self.apos.cache.get('exportPaths', id); + await self.remove(exportPath); }, - async getExportPathId(path) { - return self.apos.cache.get('exportPathIds', path); + // FIXME: [Error: EPERM: operation not permitted, unlink '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9'] { + // errno: -1, + // code: 'EPERM', + // syscall: 'unlink', + // path: '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9' + // } + async remove(filepath) { + console.log('🚀 ~ remove ~ filepath:', filepath); + try { + const stat = await fsp.lstat(filepath); + console.log('🚀 ~ remove ~ stat:', stat); + if (stat.isDirectory()) { + await fsp.rm(filepath, { + recursive: true, + force: true + }); + } else { + await fsp.unlink(filepath); + } + } catch (err) { + console.trace(); + self.apos.util.error( + `Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.` + ); + } } - }; };