Feat: mapeoProject.importConfig

package.json

@@ -114,6 +114,7 @@
     "@mapeo/schema": "3.0.0-next.13",
     "@mapeo/sqlite-indexer": "1.0.0-alpha.8",
     "@sinclair/typebox": "^0.29.6",
+    "@types/yauzl-promise": "^2.1.5",
     "b4a": "^1.6.3",
     "better-sqlite3": "^8.7.0",
     "big-sparse-array": "^1.0.3",
@@ -145,6 +146,7 @@
     "throttle-debounce": "^5.0.0",
     "tiny-typed-emitter": "^2.1.0",
     "type-fest": "^4.5.0",
-    "varint": "^6.0.0"
+    "varint": "^6.0.0",
+    "yauzl-promise": "^4.0.0"
   }
 }

src/config-import.js

@@ -0,0 +1,271 @@
+import yauzl from 'yauzl-promise'
+import { validate, valueSchemas } from '@mapeo/schema'
+import { json, buffer } from 'node:stream/consumers'
+import path from 'node:path'
+
+// Throw error if a zipfile contains more than 10,000 entries
+const MAX_ENTRIES = 10_000
+const MAX_ICON_SIZE = 10_000_000
+
+/**
+ * @typedef {yauzl.Entry} Entry
+ */
+/**
+ * @typedef {{
+ *   presets: { [id: string]: unknown }
+ *   fields:  { [id: string]: unknown }
+ * }} PresetsFile
+ */
+/**
+ * @typedef {Parameters<import('./icon-api.js').IconApi['create']>[0]} IconData
+ */
+
+/**
+ * @param {string} configPath
+ */
+export async function readConfig(configPath) {
+  /** @type {Error[]} */
+  const warnings = []
+  /** @type {yauzl.ZipFile} */
+  let iconsZip
+
+  const presetsZip = await yauzl.open(configPath)
+  if (presetsZip.entryCount > MAX_ENTRIES) {
+    // MAX_ENTRIES in MAC can be inacurrate
+    throw new Error(`Zip file contains too many entries. Max is ${MAX_ENTRIES}`)
+  }
+  /** @type {undefined | Entry} */
+  let presetsEntry
+  for await (const entry of presetsZip) {
+    if (entry.filename === 'presets.json') {
+      presetsEntry = entry
+      break
+    }
+  }
+  if (!presetsEntry) {
+    throw new Error('Zip file does not contain presets.json')
+  }
+  const presetsFile = await json(await presetsEntry.openReadStream())
+  validatePresetsFile(presetsFile)
+
+  return {
+    get warnings() {
+      return warnings
+    },
+
+    async close() {
+      presetsZip.close()
+      iconsZip.close()
+    },
+
+    /**
+     * @returns {AsyncIterable<IconData>}
+     */
+    async *icons() {
+      /** @type {IconData | undefined} */
+      let icon
+
+      iconsZip = await yauzl.open(configPath)
+      const entries = await iconsZip.readEntries(MAX_ENTRIES)
+      for (const entry of entries) {
+        if (!entry.filename.match(/^icons\/([^/]+)$/)) continue
+        if (entry.uncompressedSize > MAX_ICON_SIZE) {
+          warnings.push(
+            new Error(
+              `icon ${entry.filename} is bigger than maximum allowed size (10MB) `
+            )
+          )
+          continue
+        }
+        const buf = await buffer(await entry.openReadStream())
+        const iconFilename = entry.filename.replace(/^icons\//, '')
+        try {
+          const { name, variant } = parseIcon(iconFilename, buf)
+          if (!icon) {
+            icon = {
+              name,
+              variants: [variant],
+            }
+          } else if (icon.name === name) {
+            icon.variants.push(variant)
+          } else {
+            yield icon
+            icon = {
+              name,
+              variants: [variant],
+            }
+          }
+        } catch (err) {
+          warnings.push(err)
+        }
+      }
+      if (icon) {
+        yield icon
+      }
+    },
+
+    /**
+     * @returns {Iterable<{ name: string, value: import('@mapeo/schema').FieldValue }>}
+     */
+    *fields() {
+      const { fields } = presetsFile
+      for (const [name, field] of Object.entries(fields)) {
+        if (!isRecord(field) || !hasOwn(field, 'key')) {
+          warnings.push(new Error(`Invalid field ${name}`))
+          continue
+        }
+        /** @type {Record<string, unknown>} */
+        const fieldValue = {
+          schemaName: 'field',
+          tagKey: field.key,
+        }
+        for (const key of Object.keys(valueSchemas.field.properties)) {
+          if (hasOwn(field, key)) {
+            fieldValue[key] = field[key]
+          }
+        }
+        if (!validate('field', fieldValue)) {
+          warnings.push(new Error(`Invalid field ${name}`))
+          continue
+        }
+        yield {
+          name,
+          value: fieldValue,
+        }
+      }
+    },
+
+    /**
+     * @returns {Iterable<{ fieldNames: string[], iconName: string | undefined, value: import('@mapeo/schema').PresetValue }>}
+     */
+    *presets() {
+      const { presets } = presetsFile
+      // sort presets using the sort field, turn them into an array
+      /** @type {Array<Record<string, unknown>>} */
+      const sortedPresets = []
+      for (const [presetName, preset] of Object.entries(presets)) {
+        if (isRecord(preset)) {
+          sortedPresets.push(preset)
+        } else {
+          warnings.push(new Error(`invalid preset ${presetName}`))
+        }
+      }
+      sortedPresets.sort((preset, nextPreset) => {
+        const sort = typeof preset.sort === 'number' ? preset.sort : Infinity
+        const nextSort =
+          typeof nextPreset.sort === 'number' ? nextPreset.sort : Infinity
+        return sort - nextSort
+      })
+
+      // 5. for each preset get the corresponding fieldId and iconId, add them to the db
+      for (let preset of sortedPresets) {
+        /** @type {Record<string, unknown>} */
+        const presetValue = {
+          schemaName: 'preset',
+          fieldIds: [],
+          addTags: {},
+          removeTags: {},
+          terms: [],
+        }
+        for (const key of Object.keys(valueSchemas.preset.properties)) {
+          if (hasOwn(preset, key)) {
+            presetValue[key] = preset[key]
+          }
+        }
+        if (!validate('preset', presetValue)) {
+          warnings.push(new Error(`Invalid preset ${preset.name}`))
+          continue
+        }
+        yield {
+          fieldNames:
+            'fields' in preset && Array.isArray(preset.fields)
+              ? preset.fields
+              : [],
+          iconName:
+            'icon' in preset && typeof preset.icon === 'string'
+              ? preset.icon
+              : undefined,
+          value: presetValue,
+        }
+      }
+    },
+  }
+}
+
+/**
+ * @param {string} filename
+ * @param {Buffer} buf
+ * @returns {{ name: string, variant: IconData['variants'][Number] }}}
+ */
+function parseIcon(filename, buf) {
+  const parsedFilename = path.parse(filename)
+  const matches = parsedFilename.base.match(
+    /([a-zA-Z0-9-]+)-([a-zA-Z]+)@(\d+)x\.[a-zA-Z]+$/
+  )
+  if (!matches) {
+    throw new Error(`Unexpected icon filename ${filename}`)
+  }
+  /* eslint-disable no-unused-vars */
+  const [_, name, size, pixelDensityStr] = matches
+  const pixelDensity = Number(pixelDensityStr)
+  if (!(pixelDensity === 1 || pixelDensity === 2 || pixelDensity === 3)) {
+    throw new Error(`Error loading icon. invalid pixel density ${pixelDensity}`)
+  }
+  if (!(size === 'small' || size === 'medium' || size === 'large')) {
+    throw new Error(`Error loading icon. invalid size ${size}`)
+  }
+  if (!name) {
+    throw new Error('Error loading icon. missing name')
+  }
+  /** @type {'image/png' | 'image/svg+xml'} */
+  let mimeType
+  switch (parsedFilename.ext.toLowerCase()) {
+    case '.png':
+      mimeType = 'image/png'
+      break
+    case '.svg':
+      mimeType = 'image/svg+xml'
+      break
+    default:
+      throw new Error(`Unexpected icon extension ${parsedFilename.ext}`)
+  }
+  return {
+    name,
+    variant: {
+      size,
+      mimeType,
+      pixelDensity,
+      blob: buf,
+    },
+  }
+}
+
+/**
+ * @param {unknown} presetsFile
+ * @returns {asserts presetsFile is PresetsFile}
+ */
+function validatePresetsFile(presetsFile) {
+  if (
+    !isRecord(presetsFile) ||
+    !isRecord(presetsFile.presets) ||
+    !isRecord(presetsFile.fields)
+  ) {
+    throw new Error('Invalid presets.json file')
+  }
+}
+
+/**
+ * @param {unknown} value
+ * @returns {value is Record<string, unknown>}
+ */
+function isRecord(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value)
+}
+
+/**
+ * @param {Record<string | symbol, unknown>} obj
+ * @param {string | symbol} prop
+ */
+function hasOwn(obj, prop) {
+  return Object.prototype.hasOwnProperty.call(obj, prop)
+}

src/mapeo-project.js

@@ -45,6 +45,7 @@ import { MemberApi } from './member-api.js'
 import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js'
 import { Logger } from './logger.js'
 import { IconApi } from './icon-api.js'
+import { readConfig } from './config-import.js'
 
 /** @typedef {Omit<import('@mapeo/schema').ProjectSettingsValue, 'schemaName'>} EditableProjectSettings */
 
@@ -632,6 +633,78 @@ export class MapeoProject extends TypedEmitter {
     // 4. Assign LEFT role for device
     await this.#roles.assignRole(this.#deviceId, LEFT_ROLE_ID)
   }
+
+  /** @param {Object} opts
+   *  @param {string} opts.configPath
+   *  @returns {Promise<Error[]>}
+   */
+  async importConfig({ configPath }) {
+    // check for already present fields and presets and delete them if exist
+    await deleteAll(this.preset)
+    await deleteAll(this.field)
+
+    const config = await readConfig(configPath)
+    /** @type {Map<string, string>} */
+    const iconNameToId = new Map()
+    /** @type {Map<string, string>} */
+    const fieldNameToId = new Map()
+
+    // Do this in serial not parallel to avoid memory issues (avoid keeping all icon buffers in memory)
+    for await (const icon of config.icons()) {
+      const iconId = await this.#iconApi.create(icon)
+      iconNameToId.set(icon.name, iconId)
+    }
+
+    // Ok to create fields and presets in parallel
+    const fieldPromises = []
+    for (const { name, value } of config.fields()) {
+      fieldPromises.push(
+        this.#dataTypes.field.create(value).then(({ docId }) => {
+          fieldNameToId.set(name, docId)
+        })
+      )
+    }
+    await Promise.all(fieldPromises)
+
+    const presetsWithRefs = []
+    for (const { fieldNames, iconName, value } of config.presets()) {
+      const fieldIds = fieldNames.map((fieldName) => {
+        const id = fieldNameToId.get(fieldName)
+        if (!id) {
+          throw new Error(
+            `field ${fieldName} not found (referenced by preset ${value.name})})`
+          )
+        }
+        return id
+      })
+      presetsWithRefs.push({
+        ...value,
+        iconId: iconName && iconNameToId.get(iconName),
+        fieldIds,
+      })
+    }
+
+    // close the zip handles after we know we won't be needing them anymore
+    await config.close()
+
+    const presetPromises = presetsWithRefs.map((preset) =>
+      this.preset.create(preset)
+    )
+    const createdPresets = await Promise.all(presetPromises)
+    const presetIds = createdPresets.map(({ docId }) => docId)
+
+    await this.$setProjectSettings({
+      defaultPresets: {
+        point: presetIds,
+        line: [],
+        area: [],
+        vertex: [],
+        relation: [],
+      },
+    })
+
+    return config.warnings
+  }
 }
 
 /**
@@ -644,6 +717,16 @@ function extractEditableProjectSettings(projectDoc) {
   return result
 }
 
+// TODO: maybe a better signature than a bunch of any?
+/** @param {DataType<any,any,any,any,any>} dataType */
+async function deleteAll(dataType) {
+  const deletions = []
+  for (const { versionId } of await dataType.getMany()) {
+    deletions.push(dataType.delete(versionId))
+  }
+  return Promise.all(deletions)
+}
+
 /**
  * Return a map of namespace -> core keypair
  *