From c60b92c6f56e8d0bd6ffe5f3f411644c48439824 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 28 Nov 2023 21:31:41 +0900 Subject: [PATCH] feat: remove project.ready() (#392) * WIP initial work * rename Rpc to LocalPeers * Handle deviceInfo internally, id -> deviceId * Tests for stream error handling * remove unnecessary constructor * return replication stream * Attach protomux instance to peer info * rename and re-organize * revert changes outside scope of PR * WIP initial work * Tie everything together * rename getProjectInstance * feat: client.listLocalPeers() & `local-peers` evt * feat: add $sync API methods For now this simplifies the API (because we are only supporting local sync, not remote sync over the internet) to: - `project.$sync.getState()` - `project.$sync.start()` - `project.$sync.stop()` - Events - `sync-state` It's currently not possible to stop local discovery, nor is it possible to stop sync of the metadata namespaces (auth, config, blobIndex). The start and stop methods stop the sync of the data and blob namespaces. Fixes #134. Stacked on #360, #358 and #356. * feat: Add project.$waitForInitialSync() method Fixes Add project method to download auth + config cores #233 Rather than call this inside the `client.addProject()` method, instead I think it is better for the API consumer to call `project.$waitForInitialSync()` after adding a project, since this allows the implementer to give user feedback about what is happening. * Wait for initial sync within addProject() * fix: don't add core bitfield until core is ready * feat: expose deviceId on coreManager * fix: wait for project.ready() in waitForInitialSync * fix: skip waitForSync in tests * don't enable/disable namespace if not needed * start core download when created via sparse: false * Add debug logging This was a big lift, but necessary to be able to debug sync issues since temporarily adding console.log statements was too much work, and debugging requires knowing the deviceId associated with each message. * fix timeout * fix: Add new cores to the indexer (!!!) This caused a day of work: a bug from months back * remove unnecessary log stmt * get capabilities.getMany() to include creator * fix invite test * keep blob cores sparse * optional param for LocalPeers * re-org sync and replication Removes old replication code attached to CoreManager Still needs tests to be updated * update package-lock * chore: Add debug logging * Add new logger to discovery + dnssd * Get invite test working * fix manager logger * cleanup invite test (and make it fail :( * fix: handle duplicate connections to LocalPeers * fix stream close before channel open * send invite to non-existent peer * fixed fake timers implementation for tests * new tests for duplicate connections * cleanup and small fix * Better state debug logging * chain of invites test * fix max listeners and add skipped test * fix: only request a core key from one peer Reduces the number of duplicate requests for the same keys. * cleanup members tests with new helprs * wait for project ready when adding * only create 4 clients for chain of invites test * add e2e sync tests * add published @mapeo/mock-data * fix: don't open cores in sparse mode Turns out this changes how core.length etc. work, which confuses things * fix: option to skip auto download for tests * e2e test for stop-start sync * fix coreManager unit tests * fix blob store tests * fix discovery-key event * add coreCount to sync state * test sync with blocked peer & fix bugs * fix datatype unit tests * fix blobs server unit tests * remote peer-sync-controller unit test This is now tested in e2e tests * fix type issues caused by bad lockfile * ignore debug type errors * fixes for review comments * move utils-new into utils * Add debug info to test that sometimes fails * Update package-lock.json version * remove project.ready() (breaks things) * wait for coreOwnership write before returning --------- Co-authored-by: Andrew Chou --- src/core-ownership.js | 29 +++++++++++++++++++++++-- src/datatype/index.d.ts | 3 +++ src/datatype/index.js | 4 ++++ src/mapeo-manager.js | 2 -- src/mapeo-project.js | 43 +++++++++----------------------------- test-e2e/capabilities.js | 2 -- test-e2e/core-ownership.js | 1 - test-e2e/device-info.js | 5 ----- test-e2e/media-server.js | 4 ---- test-e2e/members.js | 4 ---- 10 files changed, 44 insertions(+), 53 deletions(-) diff --git a/src/core-ownership.js b/src/core-ownership.js index fa29ebd37..db96e124c 100644 --- a/src/core-ownership.js +++ b/src/core-ownership.js @@ -8,6 +8,7 @@ import { kTable, kSelect, kCreateWithDocId } from './datatype/index.js' import { eq, or } from 'drizzle-orm' import mapObject from 'map-obj' import { discoveryKey } from 'hypercore-crypto' +import pDefer from 'p-defer' /** * @typedef {import('./types.js').CoreOwnershipWithSignatures} CoreOwnershipWithSignatures @@ -15,6 +16,7 @@ import { discoveryKey } from 'hypercore-crypto' export class CoreOwnership { #dataType + #ownershipWriteDone /** * * @param {object} opts @@ -25,9 +27,30 @@ export class CoreOwnership { * import('@mapeo/schema').CoreOwnership, * import('@mapeo/schema').CoreOwnershipValue * >} opts.dataType + * @param {Record} opts.coreKeypairs + * @param {import('./types.js').KeyPair} opts.identityKeypair */ - constructor({ dataType }) { + constructor({ dataType, coreKeypairs, identityKeypair }) { this.#dataType = dataType + const authWriterCore = dataType.writerCore + const deferred = pDefer() + this.#ownershipWriteDone = deferred.promise + + const writeOwnership = () => { + if (authWriterCore.length > 0) { + deferred.resolve() + return + } + this.#writeOwnership(identityKeypair, coreKeypairs) + .then(deferred.resolve) + .catch(deferred.reject) + } + // @ts-ignore - opened missing from types + if (authWriterCore.opened) { + writeOwnership() + } else { + authWriterCore.on('ready', writeOwnership) + } } /** @@ -35,6 +58,7 @@ export class CoreOwnership { * @returns {Promise} deviceId of device that owns the core */ async getOwner(coreId) { + await this.#ownershipWriteDone const table = this.#dataType[kTable] const expressions = [] for (const namespace of NAMESPACES) { @@ -57,6 +81,7 @@ export class CoreOwnership { * @returns {Promise} coreId of core belonging to `deviceId` for `namespace` */ async getCoreId(deviceId, namespace) { + await this.#ownershipWriteDone const result = await this.#dataType.getByDocId(deviceId) return result[`${namespace}CoreId`] } @@ -66,7 +91,7 @@ export class CoreOwnership { * @param {import('./types.js').KeyPair} identityKeypair * @param {Record} coreKeypairs */ - async writeOwnership(identityKeypair, coreKeypairs) { + async #writeOwnership(identityKeypair, coreKeypairs) { /** @type {import('./types.js').CoreOwnershipWithSignaturesValue} */ const docValue = { schemaName: 'coreOwnership', diff --git a/src/datatype/index.d.ts b/src/datatype/index.d.ts index d12458cea..9397857b8 100644 --- a/src/datatype/index.d.ts +++ b/src/datatype/index.d.ts @@ -11,6 +11,7 @@ import { import { type BetterSQLite3Database } from 'drizzle-orm/better-sqlite3' import { SQLiteSelectBuilder } from 'drizzle-orm/sqlite-core' import { RunResult } from 'better-sqlite3' +import type Hypercore from 'hypercore' type MapeoDocTableName = `${MapeoDoc['schemaName']}Table` type GetMapeoDocTables = T[keyof T & MapeoDocTableName] @@ -59,6 +60,8 @@ export class DataType< get [kTable](): TTable + get writerCore(): Hypercore<'binary', Buffer> + [kCreateWithDocId]( docId: string, value: diff --git a/src/datatype/index.js b/src/datatype/index.js index 698c9f58b..2d8ef410f 100644 --- a/src/datatype/index.js +++ b/src/datatype/index.js @@ -91,6 +91,10 @@ export class DataType { return this.#table } + get writerCore() { + return this.#dataStore.writerCore + } + /** * @template {import('type-fest').Exact} T * @param {T} value diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 660ba2717..b2b55d3b8 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -498,7 +498,6 @@ export class MapeoManager extends TypedEmitter { try { // 4. Write device info into project const project = await this.getProject(projectPublicId) - await project.ready() try { const deviceInfo = await this.getDeviceInfo() @@ -548,7 +547,6 @@ export class MapeoManager extends TypedEmitter { * @returns {Promise} */ async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) { - await project.ready() const [capability, projectSettings] = await Promise.all([ project.$getOwnCapabilities(), project.$getProjectSettings(), diff --git a/src/mapeo-project.js b/src/mapeo-project.js index e3ada1a75..335af9bdf 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -4,7 +4,6 @@ import Database from 'better-sqlite3' import { decodeBlockPrefix } from '@mapeo/schema' import { drizzle } from 'drizzle-orm/better-sqlite3' import { migrate } from 'drizzle-orm/better-sqlite3/migrator' -import pDefer from 'p-defer' import { discoveryKey } from 'hypercore-crypto' import { CoreManager, NAMESPACES } from './core-manager/index.js' @@ -61,7 +60,6 @@ export class MapeoProject { #blobStore #coreOwnership #capabilities - #ownershipWriteDone #memberApi #iconApi #syncApi @@ -220,9 +218,16 @@ export class MapeoProject { db, }), } - + const identityKeypair = keyManager.getIdentityKeypair() + const coreKeypairs = getCoreKeypairs({ + projectKey, + projectSecretKey, + keyManager, + }) this.#coreOwnership = new CoreOwnership({ dataType: this.#dataTypes.coreOwnership, + coreKeypairs, + identityKeypair, }) this.#capabilities = new Capabilities({ dataType: this.#dataTypes.role, @@ -302,27 +307,6 @@ export class MapeoProject { this.#syncApi[kHandleDiscoveryKey](discoveryKey, stream) }) - ///////// 5. Write core ownership record - - const deferred = pDefer() - // Avoid uncaught rejection. If this is rejected then project.ready() will reject - deferred.promise.catch(() => {}) - this.#ownershipWriteDone = deferred.promise - - const authCore = this.#coreManager.getWriterCore('auth').core - authCore.on('ready', () => { - if (authCore.length > 0) return - const identityKeypair = keyManager.getIdentityKeypair() - const coreKeypairs = getCoreKeypairs({ - projectKey, - projectSecretKey, - keyManager, - }) - this.#coreOwnership - .writeOwnership(identityKeypair, coreKeypairs) - .then(deferred.resolve) - .catch(deferred.reject) - }) this.#l.log('Created project instance %h', projectKey) } @@ -355,13 +339,6 @@ export class MapeoProject { return this.#deviceId } - /** - * Resolves when hypercores have all loaded - */ - async ready() { - await Promise.all([this.#coreManager.ready(), this.#ownershipWriteDone]) - } - /** * @param {import('multi-core-indexer').Entry[]} entries * @param {{projectIndexWriter: IndexWriter, sharedIndexWriter: IndexWriter}} indexWriters @@ -544,11 +521,11 @@ function extractEditableProjectSettings(projectDoc) { * @param {Buffer} opts.projectKey * @param {Buffer} [opts.projectSecretKey] * @param {import('@mapeo/crypto').KeyManager} opts.keyManager - * @returns {Record} + * @returns {Record} */ function getCoreKeypairs({ projectKey, projectSecretKey, keyManager }) { const keypairs = - /** @type {Record} */ ({}) + /** @type {Record} */ ({}) for (const namespace of NAMESPACES) { keypairs[namespace] = diff --git a/test-e2e/capabilities.js b/test-e2e/capabilities.js index 984ff049c..30485dd1a 100644 --- a/test-e2e/capabilities.js +++ b/test-e2e/capabilities.js @@ -66,7 +66,6 @@ test('New device without capabilities', async (t) => { { waitForSync: false } ) const project = await manager.getProject(projectId) - await project.ready() const ownCapabilities = await project.$getOwnCapabilities() @@ -147,7 +146,6 @@ test('getMany() - on newly invited device before sync', async (t) => { { waitForSync: false } ) const project = await manager.getProject(projectId) - await project.ready() const expected = { [deviceId]: NO_ROLE_CAPABILITIES, diff --git a/test-e2e/core-ownership.js b/test-e2e/core-ownership.js index 97ec86061..910d4adb2 100644 --- a/test-e2e/core-ownership.js +++ b/test-e2e/core-ownership.js @@ -22,7 +22,6 @@ test('CoreOwnership', async (t) => { const projectId = await manager.createProject() const project = await manager.getProject(projectId) const coreOwnership = project[kCoreOwnership] - await project.ready() const identityKeypair = km.getIdentityKeypair() const deviceId = identityKeypair.publicKey.toString('hex') diff --git a/test-e2e/device-info.js b/test-e2e/device-info.js index 0ac3db0c4..25ebcb46c 100644 --- a/test-e2e/device-info.js +++ b/test-e2e/device-info.js @@ -45,8 +45,6 @@ test('device info written to projects', (t) => { const projectId = await manager.createProject() const project = await manager.getProject(projectId) - await project.ready() - const me = await project.$member.getById(project.deviceId) st.is(me.deviceId, project.deviceId) @@ -74,8 +72,6 @@ test('device info written to projects', (t) => { const project = await manager.getProject(projectId) - await project.ready() - const me = await project.$member.getById(project.deviceId) st.alike({ name: me.name }, { name: 'mapeo' }) @@ -101,7 +97,6 @@ test('device info written to projects', (t) => { const projects = await Promise.all( projectIds.map(async (projectId) => { const project = await manager.getProject(projectId) - await project.ready() return project }) ) diff --git a/test-e2e/media-server.js b/test-e2e/media-server.js index f7f68b2a3..9bc784156 100644 --- a/test-e2e/media-server.js +++ b/test-e2e/media-server.js @@ -33,8 +33,6 @@ test('retrieving blobs using url', async (t) => { const project = await manager.getProject(await manager.createProject()) - await project.ready() - const exceptionPromise1 = t.exception(async () => { await project.$blobs.getUrl({ driveId: randomBytes(32).toString('hex'), @@ -126,8 +124,6 @@ test('retrieving icons using url', async (t) => { const project = await manager.getProject(await manager.createProject()) - await project.ready() - const exceptionPromise1 = t.exception(async () => { await project.$icons.getIconUrl(randomBytes(32).toString('hex'), { mimeType: 'image/png', diff --git a/test-e2e/members.js b/test-e2e/members.js index 2934a531a..0a98dd45a 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -21,7 +21,6 @@ test('getting yourself after creating project', async (t) => { const deviceInfo = await manager.getDeviceInfo() const project = await manager.getProject(await manager.createProject()) - await project.ready() const me = await project.$member.getById(project.deviceId) @@ -62,7 +61,6 @@ test('getting yourself after adding project (but not yet synced)', async (t) => { waitForSync: false } ) ) - await project.ready() const me = await project.$member.getById(project.deviceId) @@ -98,7 +96,6 @@ test('getting invited member after invite rejected', async (t) => { const projectId = await invitor.createProject() const project = await invitor.getProject(projectId) - await project.ready() await invite({ invitor, @@ -131,7 +128,6 @@ test('getting invited member after invite accepted', async (t) => { const { name: inviteeName } = await invitee.getDeviceInfo() const projectId = await invitor.createProject() const project = await invitor.getProject(projectId) - await project.ready() await invite({ invitor,