Skip to content

Commit

Permalink
import: separate concerns between apos related stuff and formats
Browse files Browse the repository at this point in the history
  • Loading branch information
Etienne Laurent committed Apr 5, 2024
1 parent 8c35515 commit 0f015b3
Show file tree
Hide file tree
Showing 3 changed files with 168 additions and 114 deletions.
16 changes: 12 additions & 4 deletions lib/formats/csv.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ module.exports = {
// TODO: remove these? because it really messes with the duplicated docs detection system
// importVersions: [ 'draft' ],
// exportVersions: [ 'published' ],
async import(exportPath) {
async import(exportPath, { docIds } = {}) {
const reader = fs.createReadStream(exportPath);
const parser = reader
.pipe(
Expand Down Expand Up @@ -48,16 +48,24 @@ module.exports = {
reader.on('error', reject);
parser.on('error', reject);
parser.on('end', () => {
console.log('IMPORT RETURNS:');
console.dir({
docs: !docIds
? docs
: docs.filter(({ aposDocId }) => docIds.includes(aposDocId)),
exportPath
}, { depth: 9 });
resolve({
docs: !docIds
? docs
: docs.filter(({ aposDocId }) => docIds.includes(aposDocId))
: docs.filter(({ aposDocId }) => docIds.includes(aposDocId)),
exportPath
});
});
});
},
async export(filepath, { docs }) {
const writer = fs.createWriteStream(filepath);
async export(exportPath, { docs }) {
const writer = fs.createWriteStream(exportPath);
const stringifier = stringify({
header: true,
columns: getColumnsNames(docs),
Expand Down
145 changes: 93 additions & 52 deletions lib/formats/gzip.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,42 +17,59 @@ module.exports = {
includeAttachments: true,
// importVersions: [ 'draft', 'published' ],
// exportVersions: [ 'draft', 'published' ],
async import(filepath) {
const extractPath = filepath.replace(this.allowedExtension, '');
async import(exportPath, { docIds } = {}) {
console.log('🚀 ~ import ~ exportPath:', exportPath);
console.log('🚀 ~ import ~ docIds:', docIds);
console.log('this.allowedExtension', this.allowedExtension);
// If the given path is actually the archive, we first need to extract it.
// Then we no longer need the archive file, so we remove it.
if (exportPath.endsWith(this.allowedExtension)) {
const filepath = exportPath;
exportPath = exportPath.replace(this.allowedExtension, '');
await extract(filepath, exportPath);
await remove(filepath);

if (!fs.existsSync(extractPath)) {
await fsp.mkdir(extractPath);
}
console.log('🚀 ~ import ~ exportPath:', exportPath);

const readStream = fs.createReadStream(filepath);
const gunzip = zlib.createGunzip();
const extract = tar.extract();

readStream
.pipe(gunzip)
.pipe(extract);

return new Promise((resolve, reject) => {
readStream.on('error', reject);
gunzip.on('error', reject);
extract.on('error', reject);

extract.on('entry', (header, stream, next) => {
if (header.type === 'directory') {
fsp
.mkdir(`${extractPath}/${header.name}`)
.then(next)
.catch(reject);
} else {
stream.pipe(fs.WriteStream(`${extractPath}/${header.name}`));
stream.on('end', next);
const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json'));
const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json'));
console.log('IMPORT RETURNS:');
console.dir({
docs: !docIds
? EJSON.parse(docs)
: EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)),
attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({
attachment,
file: {
name: `${attachment.name}.${attachment.extension}`,
path: path.join(
exportPath,
'attachments',
`${attachment._id}-${attachment.name}.${attachment.extension}`
)
}
});
})),
exportPath
}, { depth: 9 });

extract.on('finish', () => {
resolve(extractPath);
});
});
return {
docs: !docIds
? EJSON.parse(docs)
: EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)),
attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({
attachment,
file: {
name: `${attachment.name}.${attachment.extension}`,
path: path.join(
exportPath,
'attachments',
`${attachment._id}-${attachment.name}.${attachment.extension}`
)
}
})),
exportPath
};
},
async export(
filepath,
Expand Down Expand Up @@ -117,30 +134,54 @@ module.exports = {
})
.catch(reject);
});
},
async read(exportPath, docIds) {
const docs = await fsp.readFile(path.join(exportPath, 'aposDocs.json'));
const attachments = await fsp.readFile(path.join(exportPath, 'aposAttachments.json'));

return {
docs: !docIds
? EJSON.parse(docs)
: EJSON.parse(docs).filter(({ aposDocId }) => docIds.includes(aposDocId)),
attachmentsInfo: EJSON.parse(attachments).map((attachment) => ({
attachment,
file: {
name: `${attachment.name}.${attachment.extension}`,
path: path.join(
exportPath,
'attachments',
`${attachment._id}-${attachment.name}.${attachment.extension}`
)
}
}))
};
}
};

async function extract(filepath, exportPath) {
console.log('🚀 ~ extract ~ filepath:', filepath);
console.log('🚀 ~ extract ~ exportPath:', exportPath);

if (!fs.existsSync(exportPath)) {
await fsp.mkdir(exportPath);
}

const readStream = fs.createReadStream(filepath);
const gunzip = zlib.createGunzip();
const extract = tar.extract();

readStream
.pipe(gunzip)
.pipe(extract);

return new Promise((resolve, reject) => {
readStream.on('error', reject);
gunzip.on('error', reject);
extract.on('error', reject);

extract.on('entry', (header, stream, next) => {
if (header.type === 'directory') {
fsp
.mkdir(path.join(exportPath, header.name))
.then(next)
.catch(reject);
} else {
stream.pipe(fs.WriteStream(path.join(exportPath, header.name)));
stream.on('end', next);
}
});
extract.on('finish', resolve);
});
}

async function remove(filepath) {
console.log('🚀 ~ remove ~ filepath:', filepath);
try {
await fsp.unlink(filepath);
} catch (error) {
self.apos.util.error(error);
}
}

function addTarEntry(pack, options, data = null) {
return new Promise((resolve, reject) => {
pack.entry(options, data, error => {
Expand Down
121 changes: 63 additions & 58 deletions lib/methods/import.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,19 +26,23 @@ module.exports = self => {
let exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId));

let docs;
let attachmentsInfo = [];
let attachmentsInfo;

try {
if (!exportPath) {
exportPath = await format.input(file.path);
if (exportPath) {
({
docs = [],
attachmentsInfo = []
} = await format.import(exportPath));
} else {
({
docs = [],
attachmentsInfo = [],
exportPath
} = await format.import(file.path));

await self.setExportPathId(exportPath);
}
const filesData = await format.getFilesData(exportPath);

docs = filesData.docs;
attachmentsInfo = filesData.attachmentsInfo;

await self.cleanFile(file.path);
} catch (error) {
await self.apos.notify(req, 'aposImportExport:importFileError', {
interpolate: { format: format.label },
Expand Down Expand Up @@ -85,9 +89,7 @@ module.exports = self => {
self.rewriteDocsWithCurrentLocale(req, docs);
}

const total = format.includeAttachments
? docs.length + attachmentsInfo.length
: docs.length;
const total = docs.length + attachmentsInfo.length;

const {
reporting, jobId, notificationId
Expand All @@ -97,16 +99,12 @@ module.exports = self => {
duplicatedDocs, duplicatedIds, failedIds
} = await self.insertDocs(req, docs, reporting);

let importedAttachments = [];

if (format.includeAttachments) {
importedAttachments = await self.insertAttachments(req, {
attachmentsInfo,
reporting,
duplicatedIds,
docIds: new Set(docs.map(({ aposDocId }) => aposDocId))
});
}
const importedAttachments = await self.insertAttachments(req, {
attachmentsInfo,
reporting,
duplicatedIds,
docIds: new Set(docs.map(({ aposDocId }) => aposDocId))
});

if (!duplicatedDocs.length) {
await reporting.end();
Expand All @@ -131,7 +129,7 @@ module.exports = self => {
self.apos.util.error(error);
});

await self.cleanFile(exportPath);
await self.remove(exportPath);
return;
}

Expand Down Expand Up @@ -417,25 +415,6 @@ module.exports = self => {
}
},

async cleanFile(path) {
try {
const stat = await fsp.lstat(path);
if (stat.isDirectory()) {
await fsp.rm(path, {
recursive: true,
force: true
});
} else {
await fsp.unlink(path);
}
} catch (err) {
console.trace();
self.apos.util.error(
`Error while trying to remove the file or folder: ${path}. You might want to remove it yourself.`
);
}
},

async overrideDuplicates(req) {
const overrideLocale = self.apos.launder.boolean(req.body.overrideLocale);
const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId));
Expand All @@ -458,7 +437,7 @@ module.exports = self => {
throw self.apos.error(`invalid format "${formatLabel}"`);
}

const { docs, attachmentsInfo } = await format.getFilesData(exportPath, docIds);
const { docs, attachmentsInfo } = await format.import(exportPath, docIds);

const differentDocsLocale = self.getFirstDifferentLocale(req, docs);
const siteHasMultipleLocales = Object.keys(self.apos.i18n.locales).length > 1;
Expand Down Expand Up @@ -521,6 +500,21 @@ module.exports = self => {
}
},

async setExportPathId(path) {
const id = self.apos.util.generateId();
await self.apos.cache.set('exportPaths', id, path, 86400);
await self.apos.cache.set('exportPathIds', path, id, 86400);
return id;
},

async getExportPathById(id) {
return self.apos.cache.get('exportPaths', id);
},

async getExportPathId(path) {
return self.apos.cache.get('exportPathIds', path);
},

async cleanExport(req) {
const exportPath = await self.getExportPathById(self.apos.launder.string(req.body.exportPathId));
if (!exportPath) {
Expand All @@ -538,23 +532,34 @@ module.exports = self => {
self.apos.notification.dismiss(req, notificationId, 2000).catch(self.apos.util.error);
}

await self.cleanFile(exportPath);
},

async setExportPathId(path) {
const id = self.apos.util.generateId();
await self.apos.cache.set('exportPaths', id, path, 86400);
await self.apos.cache.set('exportPathIds', path, id, 86400);
return id;
},

async getExportPathById(id) {
return self.apos.cache.get('exportPaths', id);
await self.remove(exportPath);
},

async getExportPathId(path) {
return self.apos.cache.get('exportPathIds', path);
// FIXME: [Error: EPERM: operation not permitted, unlink '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9'] {
// errno: -1,
// code: 'EPERM',
// syscall: 'unlink',
// path: '/var/folders/9l/mrtp2vmj6lldh2qdzxm18k_r0000gn/T/lHUaif4Sm8uyrUL5MY3lRLs9'
// }
async remove(filepath) {
console.log('🚀 ~ remove ~ filepath:', filepath);
try {
const stat = await fsp.lstat(filepath);
console.log('🚀 ~ remove ~ stat:', stat);
if (stat.isDirectory()) {
await fsp.rm(filepath, {
recursive: true,
force: true
});
} else {
await fsp.unlink(filepath);
}
} catch (err) {
console.trace();
self.apos.util.error(
`Error while trying to remove the file or folder: ${filepath}. You might want to remove it yourself.`
);
}
}

};
};

0 comments on commit 0f015b3

Please sign in to comment.