Refactor: Dependency inversion on generator

email/libmailmerge/src/engines/nunjucks-md/index.ts

@@ -2,7 +2,7 @@ import { promises as fs } from "fs";
 import nunjucks from "nunjucks";
 
 import { renderMarkdownToHtml } from "../../markdown/toHtml.js";
-import { MappedCSVRecord } from "../../util/types.js";
+import { MappedRecord } from "../../util/types.js";
 import { TemplateEngineOptions, TemplatePreviews } from "../types.js";
 import { TemplateEngine } from "../types.js";
 import getTemplateFields from "./getFields.js";
@@ -69,7 +69,7 @@ export default class NunjucksMarkdownEngine extends TemplateEngine {
         return getTemplateFields(this.loadedTemplate);
     }
 
-    public override async renderPreview(record: MappedCSVRecord) {
+    public override async renderPreview(record: MappedRecord) {
         if (!this.loadedTemplate) {
             throw new Error("Template not loaded");
         }

email/libmailmerge/src/engines/types.ts

@@ -2,7 +2,7 @@
  * Types shared by all template engines.
  * @packageDocumentation
  */
-import { MappedCSVRecord } from "../util/types";
+import { MappedRecord } from "../util/types";
 
 /**
  * Generic "any" type for template engine options: essentially a dictionary.
@@ -53,7 +53,7 @@ export abstract class TemplateEngine {
      * @param record - The record to render the template with, where the keys correspond to the fields extracted in {@link TemplateEngine.extractFields}
      * @returns A promise that resolves to an array of {@link TemplatePreviews} objects - check the type for more information
      */
-    abstract renderPreview(record: MappedCSVRecord): Promise<TemplatePreviews>;
+    abstract renderPreview(record: MappedRecord): Promise<TemplatePreviews>;
 
     /**
      * Given previews generated by {@link TemplateEngine.renderPreview} or this method, re-render a preview.
@@ -70,7 +70,7 @@ export abstract class TemplateEngine {
      */
     abstract rerenderPreviews(
         loadedPreviews: TemplatePreviews,
-        associatedRecord: MappedCSVRecord,
+        associatedRecord: MappedRecord,
     ): Promise<TemplatePreviews>;
 
     /**
@@ -80,7 +80,7 @@ export abstract class TemplateEngine {
      */
     abstract getHTMLToSend(
         loadedPreviews: TemplatePreviews,
-        associatedRecord: MappedCSVRecord,
+        associatedRecord: MappedRecord,
     ): Promise<string>;
 }
 

email/libmailmerge/src/previews/sidecarData.ts

@@ -11,7 +11,7 @@ import { join } from "path";
 import { TEMPLATE_ENGINES } from "../engines/index.js";
 import { TemplateEngineOptions, TemplatePreview, TemplatePreviews } from "../engines/types.js";
 import Mailer from "../mailer/mailer.js";
-import { MappedCSVRecord, EmailString } from "../util/types.js";
+import { MappedRecord, EmailString } from "../util/types.js";
 import { SidecarData } from "./types.js";
 
 const PARTS_SEPARATOR = "__";
@@ -26,8 +26,8 @@ const logger = createLogger("docsoc.sidecar");
  * @returns
  */
 export const getRecordPreviewPrefix = (
-    record: MappedCSVRecord,
-    fileNamer: (record: MappedCSVRecord) => string,
+    record: MappedRecord,
+    fileNamer: (record: MappedRecord) => string,
 ) => `${fileNamer(record)}`;
 
 /**
@@ -40,8 +40,8 @@ export const getRecordPreviewPrefix = (
  * // => "file_1__nunjucks__preview1.txt"
  */
 export const getRecordPreviewPrefixForIndividual = (
-    record: MappedCSVRecord,
-    fileNamer: (record: MappedCSVRecord) => string,
+    record: MappedRecord,
+    fileNamer: (record: MappedRecord) => string,
     templateEngine: string,
     preview: TemplatePreview,
 ) =>
@@ -58,16 +58,16 @@ export const getRecordPreviewPrefixForIndividual = (
  * // => "file_1-metadata.json"
  */
 export const getRecordPreviewPrefixForMetadata = (
-    record: MappedCSVRecord,
-    fileNamer: (record: MappedCSVRecord) => string,
+    record: MappedRecord,
+    fileNamer: (record: MappedRecord) => string,
 ) => `${getRecordPreviewPrefix(record, fileNamer)}${METADATA_FILE_SUFFIX}`;
 
 type ValidRecordReturn = { valid: false; reason: string } | { valid: true };
 /**
  * Check a record is valid for use in mailmerge - specifically, that it has a valid email address and a subject.
  * @param record __Mapped__ CSV Record to validate
  */
-export const validateRecord = (record: MappedCSVRecord): ValidRecordReturn => {
+export const validateRecord = (record: MappedRecord): ValidRecordReturn => {
     const validateAll = parseEmailList(record["email"] as string).reduce(
         (acc, email) => acc && Mailer.validateEmail(email),
         true,
@@ -100,9 +100,9 @@ export const validateRecord = (record: MappedCSVRecord): ValidRecordReturn => {
  * @returns
  */
 export async function writeMetadata(
-    record: MappedCSVRecord,
+    record: MappedRecord,
     sidecarData: SidecarData,
-    fileNamer: (record: MappedCSVRecord) => string,
+    fileNamer: (record: MappedRecord) => string,
     previewsRoot: string,
 ): Promise<void> {
     const recordState = validateRecord(record);
@@ -137,8 +137,8 @@ const parseEmailList = (emailList: string | undefined): EmailString[] =>
  * @returns Sidecar metadata
  */
 export function getSidecarMetadata(
-    fileNamer: (record: MappedCSVRecord) => string,
-    record: MappedCSVRecord,
+    fileNamer: (record: MappedRecord) => string,
+    record: MappedRecord,
     templateEngine: TEMPLATE_ENGINES,
     templateOptions: TemplateEngineOptions,
     attachments: string[],
@@ -161,16 +161,32 @@ export function getSidecarMetadata(
                 content: undefined,
             },
         })),
-        email: {
-            to: parseEmailList(record["email"] as string),
-            cc: parseEmailList(record["cc"] as string),
-            bcc: parseEmailList(record["bcc"] as string),
-            subject: record["subject"] as string,
-        },
+        email: createEmailData(record),
         attachments,
     };
 }
 
+export interface EmailData {
+    to: EmailString[];
+    cc: EmailString[];
+    bcc: EmailString[];
+    subject: string;
+}
+
+/**
+ * Given a __mapped__ record, extract the expected email data from it.
+ * @param record Mapped record to get email data from
+ * @returns 
+ */
+export function createEmailData(record: MappedRecord): EmailData {
+    return {
+        to: parseEmailList(record["email"] as string),
+        cc: parseEmailList(record["cc"] as string),
+        bcc: parseEmailList(record["bcc"] as string),
+        subject: record["subject"] as string,
+    };
+}
+
 /**
  * Write the sidecar metadata file for a record
  */

email/libmailmerge/src/previews/types.ts

@@ -1,6 +1,6 @@
 import { ENGINES_MAP } from "../engines/index.js";
 import { TemplateEngineOptions, TemplatePreview } from "../engines/types.js";
-import { MappedCSVRecord, EmailString } from "../util/types.js";
+import { MappedRecord, EmailString } from "../util/types.js";
 
 /**
  * Outputted to JSON files next to rendered template previews, containing metadata about the preview.
@@ -9,7 +9,7 @@ export interface SidecarData {
     /** Name of the template rendered (used for logging) */
     name: string;
     /** Record associated with the template rendered  */
-    record: MappedCSVRecord;
+    record: MappedRecord;
     /** Engine used */
     engine: keyof typeof ENGINES_MAP;
     /** Options given to the engine */

email/libmailmerge/src/util/constants.ts

@@ -1,9 +1,9 @@
 export const DOCSOC_DEFAULT_FROM_LINE = `"DoCSoc" <[email protected]>`;
 
-/** Expected names of requried email fields in CSVRecords */
-export const CSV_DEFAULT_FIELD_NAMES = {
+/** Expected names of requried email fields in records to data merge on */
+export const DEFAULT_FIELD_NAMES = {
     to: "email",
     subject: "subject",
     cc: "cc",
     bcc: "bcc",
-};
+} as const;

email/libmailmerge/src/util/types.ts

@@ -1,5 +1,5 @@
 export type EmailString = `${string}@${string}`;
 export type FromEmail = `"${string}" <${EmailString}>`;
 
-export type RawCSVRecord = Record<string, unknown>;
-export type MappedCSVRecord = Record<string, unknown>;
+export type RawRecord = Record<string, unknown>;
+export type MappedRecord = Record<string, unknown>;

email/mailmerge-cli/src/commands/generate/nunjucks.ts

@@ -1,8 +1,15 @@
 import { NunjucksMarkdownEngine, NunjucksMarkdownTemplateOptions } from "@docsoc/libmailmerge";
 import { Args, Command, Flags } from "@oclif/core";
+import { mkdirp } from "mkdirp";
 import { join } from "path";
 
+import { CSVBackend } from "../../common/dataSource.js";
 import generatePreviews, { CliOptions as GeneratePreviewsOptions } from "../../common/generate.js";
+import { JSONSidecarsBackend } from "../../common/outputBackend.js";
+import { getFileNameSchemeInteractively } from "../../interactivity/getFileNameSchemeInteractively.js";
+import { getKeysForAttachments } from "../../interactivity/getKeysForAttachments.js";
+import { getRunNameInteractively } from "../../interactivity/getRunNameInteractively.js";
+import { mapFieldsInteractive } from "../../interactivity/mapFieldsInteractive.js";
 import { DEFAULT_DIRS } from "../../util/constant.js";
 
 export default class GenerateNunjucks extends Command {
@@ -64,8 +71,10 @@ export default class GenerateNunjucks extends Command {
             templatePath: args.template,
             rootHtmlTemplate: flags.htmlTemplate,
         };
+        const runName = flags.name ?? (await getRunNameInteractively());
+        const outputRoot = join(flags.output, runName);
+        await mkdirp(outputRoot);
         const options: GeneratePreviewsOptions = {
-            csvFile: args.csvFile,
             engineInfo: {
                 options: engineOptions,
                 name: "nunjucks",
@@ -77,7 +86,15 @@ export default class GenerateNunjucks extends Command {
                 enableBCC: flags.bcc,
                 enableCC: flags.cc,
             },
-            name: flags.name,
+            dataSource: new CSVBackend(join(process.cwd(), args.csvFile)),
+            storageBackend: new JSONSidecarsBackend(outputRoot, {
+                type: "dynamic",
+                namer: getFileNameSchemeInteractively,
+            }),
+            mappings: {
+                headersToTemplateMap: mapFieldsInteractive,
+                keysForAttachments: getKeysForAttachments,
+            },
         };
         await generatePreviews(options);
     }

email/mailmerge-cli/src/common/dataSource.ts

@@ -0,0 +1,48 @@
+import { RawRecord } from "@docsoc/libmailmerge";
+import { createLogger } from "@docsoc/util";
+import { parse } from "csv-parse";
+import fs from "fs/promises";
+
+/**
+ * Generic way of loading data in to do a data merge on.
+ *
+ * All you need to do is implement the `loadRecords` method, that resolves
+ * to a set of headers and records.
+ *
+ * Headers are a set of strings, and records are an array of objects where the keys are the headers and are strings
+ *
+ * For an example, see {@link CSVBackend}
+ *
+ */
+export interface DataSource {
+    loadRecords: () => Promise<{
+        headers: Set<string>;
+        records: RawRecord[];
+    }>;
+}
+
+const logger = createLogger("csv");
+
+export class CSVBackend implements DataSource {
+    constructor(private csvFile: string) {}
+    async loadRecords(): Promise<{ headers: Set<string>; records: RawRecord[] }> {
+        // Load CSV
+        logger.info("Loading CSV...");
+        const csvRaw = await fs.readFile(this.csvFile, "utf-8");
+        logger.debug("Parsing & loading CSV...");
+        const csvParsed = parse(csvRaw, { columns: true });
+        const records: RawRecord[] = [];
+        for await (const record of csvParsed) {
+            records.push(record);
+        }
+        logger.info(`Loaded ${records.length} records`);
+        logger.debug("Extracting headers from first record's keys...");
+        if (records.length === 0) {
+            logger.error("No records found in CSV");
+            throw new Error("No records found in CSV");
+        }
+        const headers = new Set<string>(Object.keys(records[0]));
+        logger.info(`Headers: ${Object.keys(records[0]).join(", ")}`);
+        return { headers, records };
+    }
+}