From d687f1d7d926f12457ed9d0d47a40778fb7bff89 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 15:34:10 +0900 Subject: [PATCH 01/69] WIP initial work --- src/discovery/local-discovery.js | 17 +++++++++++------ src/mapeo-manager.js | 26 ++++++++++++++++++++++++++ src/rpc/index.js | 2 +- src/utils.js | 7 +++++-- 4 files changed, 43 insertions(+), 9 deletions(-) diff --git a/src/discovery/local-discovery.js b/src/discovery/local-discovery.js index 33ec0863c..6c1ade7bb 100644 --- a/src/discovery/local-discovery.js +++ b/src/discovery/local-discovery.js @@ -10,12 +10,13 @@ import pTimeout from 'p-timeout' import { keyToPublicId } from '@mapeo/crypto' /** @typedef {{ publicKey: Buffer, secretKey: Buffer }} Keypair */ +/** @typedef {import('../utils.js').OpenedNoiseStream} OpenedNoiseStream */ export const ERR_DUPLICATE = 'Duplicate connection' /** * @typedef {Object} DiscoveryEvents - * @property {(connection: import('@hyperswarm/secret-stream')) => void} connection + * @property {(connection: OpenedNoiseStream) => void} connection */ /** @@ -24,7 +25,7 @@ export const ERR_DUPLICATE = 'Duplicate connection' export class LocalDiscovery extends TypedEmitter { #identityKeypair #server - /** @type {Map>} */ + /** @type {Map} */ #noiseConnections = new Map() #dnssd #sm @@ -142,14 +143,18 @@ export class LocalDiscovery extends TypedEmitter { // Further errors will be handled in #handleNoiseStreamConnection() socket.off('error', onSocketError) secretStream.off('error', this.#handleSocketError) - this.#handleNoiseStreamConnection(secretStream) + this.#handleNoiseStreamConnection( + // We know the NoiseStream is open at this point, so we can coerce the type + /** @type {OpenedNoiseStream} */ + (secretStream) + ) }) } /** * - * @param {NoiseSecretStream} existing - * @param {NoiseSecretStream} keeping + * @param {OpenedNoiseStream} existing + * @param {OpenedNoiseStream} keeping */ #handleConnectionSwap(existing, keeping) { let closed = false @@ -174,7 +179,7 @@ export class LocalDiscovery extends TypedEmitter { /** * - * @param {NoiseSecretStream} conn + * @param {OpenedNoiseStream} conn * @returns */ #handleNoiseStreamConnection(conn) { diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 23842de7e..35adcadb2 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -17,6 +17,7 @@ import { ProjectKeys } from './generated/keys.js' import { deNullify, getDeviceId, + keyToId, projectIdToNonce, projectKeyToId, projectKeyToPublicId, @@ -24,6 +25,7 @@ import { import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' import { MapeoRPC } from './rpc/index.js' import { InviteApi } from './invite-api.js' +import { LocalDiscovery } from './discovery/local-discovery.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -50,6 +52,7 @@ export class MapeoManager { #deviceId #rpc #invite + #localDiscovery /** * @param {Object} opts @@ -103,6 +106,17 @@ export class MapeoManager { } else { this.#coreStorage = coreStorage } + + this.#localDiscovery = new LocalDiscovery({ + identityKeypair: this.#keyManager.getIdentityKeypair(), + }) + + this.#localDiscovery.on('connection', (connection) => { + this.#handleDiscoveryConnection(connection).catch((e) => { + // Ignore errors here for now + console.error('Error handling discovery connection', e) + }) + }) } /** @@ -382,6 +396,18 @@ export class MapeoManager { return projectPublicId } + /** + * @param {import('./discovery/local-discovery.js').OpenedNoiseStream} connection + */ + async #handleDiscoveryConnection(connection) { + const peerId = keyToId(connection.remotePublicKey) + this.#rpc.connect(connection) + const { name } = await this.getDeviceInfo() + if (name) { + this.#rpc.sendDeviceInfo(peerId, { name }) + } + } + /** * @template {import('type-fest').Exact} T * @param {T} deviceInfo diff --git a/src/rpc/index.js b/src/rpc/index.js index f3551c104..8bf4fc859 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -221,7 +221,7 @@ export class MapeoRPC extends TypedEmitter { /** * Connect to a peer over an existing NoiseSecretStream * - * @param {import('../types.js').NoiseStream | import('../types.js').ProtocolStream} stream a NoiseSecretStream from @hyperswarm/secret-stream + * @param {import('../types.js').NoiseStream | import('../types.js').ProtocolStream} stream a NoiseSecretStream from @hyperswarm/secret-stream */ connect(stream) { if (!stream.noiseStream) throw new Error('Invalid stream') diff --git a/src/utils.js b/src/utils.js index 92c0568f6..07fc064ea 100644 --- a/src/utils.js +++ b/src/utils.js @@ -46,9 +46,12 @@ export function truncateId(keyOrId, length = 3) { return keyToId(keyOrId).slice(0, length) } -/** @typedef {import('@hyperswarm/secret-stream')} NoiseStream */ +/** @typedef {import('@hyperswarm/secret-stream')} NoiseStream */ /** @typedef {NoiseStream & { destroyed: true }} DestroyedNoiseStream */ -/** @typedef {NoiseStream & { publicKey: Buffer, remotePublicKey: Buffer, handshake: Buffer }} OpenedNoiseStream */ +/** + * @template {import('node:stream').Duplex | import('streamx').Duplex} [T=import('node:stream').Duplex | import('streamx').Duplex] + * @typedef {import('@hyperswarm/secret-stream') & { publicKey: Buffer, remotePublicKey: Buffer, handshake: Buffer }} OpenedNoiseStream + */ /** * Utility to await a NoiseSecretStream to open, that returns a stream with the From 74a77e2787e7f64d6e9ec2700e9791097ac7f2b8 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 15:38:25 +0900 Subject: [PATCH 02/69] rename Rpc to LocalPeers --- src/invite-api.js | 2 +- src/mapeo-manager.js | 4 +- src/mapeo-project.js | 2 +- src/member-api.js | 2 +- src/rpc/index.js | 2 +- test-types/data-types.ts | 4 +- tests/helpers/rpc.js | 6 +- tests/invite-api.js | 54 +++++++++--------- tests/rpc.js | 115 +++++++++++++++++++-------------------- 9 files changed, 94 insertions(+), 97 deletions(-) diff --git a/src/invite-api.js b/src/invite-api.js index cb871d0aa..630d1f42b 100644 --- a/src/invite-api.js +++ b/src/invite-api.js @@ -47,7 +47,7 @@ export class InviteApi extends TypedEmitter { /** * @param {Object} options - * @param {import('./rpc/index.js').MapeoRPC} options.rpc + * @param {import('./rpc/index.js').LocalPeers} options.rpc * @param {object} options.queries * @param {(projectId: string) => Promise} options.queries.isMember * @param {(invite: import('./generated/rpc.js').Invite) => Promise} options.queries.addProject diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 35adcadb2..b44128b8e 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -23,7 +23,7 @@ import { projectKeyToPublicId, } from './utils.js' import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' -import { MapeoRPC } from './rpc/index.js' +import { LocalPeers } from './rpc/index.js' import { InviteApi } from './invite-api.js' import { LocalDiscovery } from './discovery/local-discovery.js' @@ -72,7 +72,7 @@ export class MapeoManager { migrationsFolder: new URL('../drizzle/client', import.meta.url).pathname, }) - this.#rpc = new MapeoRPC() + this.#rpc = new LocalPeers() this.#keyManager = new KeyManager(rootKey) this.#deviceId = getDeviceId(this.#keyManager) this.#projectSettingsIndexWriter = new IndexWriter({ diff --git a/src/mapeo-project.js b/src/mapeo-project.js index dd5bd3992..8d6af591a 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -67,7 +67,7 @@ export class MapeoProject { * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.sharedDb * @param {IndexWriter} opts.sharedIndexWriter * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data - * @param {import('./rpc/index.js').MapeoRPC} opts.rpc + * @param {import('./rpc/index.js').LocalPeers} opts.rpc * */ constructor({ diff --git a/src/member-api.js b/src/member-api.js index d79b22016..6f92659db 100644 --- a/src/member-api.js +++ b/src/member-api.js @@ -22,7 +22,7 @@ export class MemberApi extends TypedEmitter { * @param {import('./core-ownership.js').CoreOwnership} opts.coreOwnership * @param {import('./generated/keys.js').EncryptionKeys} opts.encryptionKeys * @param {Buffer} opts.projectKey - * @param {import('./rpc/index.js').MapeoRPC} opts.rpc + * @param {import('./rpc/index.js').LocalPeers} opts.rpc * @param {Object} opts.dataTypes * @param {Pick} opts.dataTypes.deviceInfo * @param {Pick} opts.dataTypes.project diff --git a/src/rpc/index.js b/src/rpc/index.js index 8bf4fc859..29e2deac0 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -127,7 +127,7 @@ class Peer { */ /** @extends {TypedEmitter} */ -export class MapeoRPC extends TypedEmitter { +export class LocalPeers extends TypedEmitter { /** @type {Map} */ #peers = new Map() /** @type {Set>} */ diff --git a/test-types/data-types.ts b/test-types/data-types.ts index f93833ad4..74cef62a9 100644 --- a/test-types/data-types.ts +++ b/test-types/data-types.ts @@ -14,7 +14,7 @@ import { drizzle } from 'drizzle-orm/better-sqlite3' import RAM from 'random-access-memory' import { IndexWriter } from '../dist/index-writer/index.js' import { projectSettingsTable } from '../dist/schema/client.js' -import { MapeoRPC } from '../dist/rpc/index.js' +import { LocalPeers } from '../dist/rpc/index.js' import { Expect, type Equal } from './utils.js' type Forks = { forks: string[] } @@ -36,7 +36,7 @@ const mapeoProject = new MapeoProject({ tables: [projectSettingsTable], sqlite, }), - rpc: new MapeoRPC(), + rpc: new LocalPeers(), }) ///// Observations diff --git a/tests/helpers/rpc.js b/tests/helpers/rpc.js index 6858e1f1f..c735096e5 100644 --- a/tests/helpers/rpc.js +++ b/tests/helpers/rpc.js @@ -5,8 +5,8 @@ import NoiseSecretStream from '@hyperswarm/secret-stream' */ /** - * @param {import('../../src/rpc/index.js').MapeoRPC} rpc1 - * @param {import('../../src/rpc/index.js').MapeoRPC} rpc2 + * @param {import('../../src/rpc/index.js').LocalPeers} rpc1 + * @param {import('../../src/rpc/index.js').LocalPeers} rpc2 * @param { {kp1?: KeyPair, kp2?: KeyPair} } [keyPairs] * @returns {() => Promise<[void, void]>} */ @@ -29,9 +29,7 @@ export function replicate( // @ts-expect-error n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream) - // @ts-expect-error rpc1.connect(n1) - // @ts-expect-error rpc2.connect(n2) return async function destroy() { diff --git a/tests/invite-api.js b/tests/invite-api.js index de24e127a..19f058dbd 100644 --- a/tests/invite-api.js +++ b/tests/invite-api.js @@ -1,7 +1,7 @@ import test from 'brittle' import { randomBytes } from 'crypto' import { KeyManager } from '@mapeo/crypto' -import { MapeoRPC } from '../src/rpc/index.js' +import { LocalPeers } from '../src/rpc/index.js' import { InviteApi } from '../src/invite-api.js' import { projectKeyToPublicId } from '../src/utils.js' import { replicate } from './helpers/rpc.js' @@ -15,7 +15,7 @@ test('invite-received event has expected payload', async (t) => { const projects = new Map() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -65,7 +65,7 @@ test('Accept invite', async (t) => { const projects = new Map() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -88,7 +88,7 @@ test('Accept invite', async (t) => { encryptionKeys, }) - t.is(response, MapeoRPC.InviteResponse.ACCEPT) + t.is(response, LocalPeers.InviteResponse.ACCEPT) }) inviteApi.on('invite-received', async ({ projectId }) => { @@ -109,7 +109,7 @@ test('Reject invite', async (t) => { const projects = new Map() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -132,7 +132,7 @@ test('Reject invite', async (t) => { encryptionKeys, }) - t.is(response, MapeoRPC.InviteResponse.REJECT) + t.is(response, LocalPeers.InviteResponse.REJECT) }) inviteApi.on('invite-received', async ({ projectId }) => { @@ -152,7 +152,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -176,7 +176,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { t.is( response, - MapeoRPC.InviteResponse.ALREADY, + LocalPeers.InviteResponse.ALREADY, 'invited peer automatically responds with "ALREADY"' ) }) @@ -195,7 +195,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() let isMember = false const inviteApi = new InviteApi({ @@ -220,7 +220,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { t.is( response, - MapeoRPC.InviteResponse.ALREADY, + LocalPeers.InviteResponse.ALREADY, 'invited peer automatically responds with "ALREADY"' ) }) @@ -242,7 +242,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { const projects = new Map() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -262,14 +262,14 @@ test('Receiving invite for project that peer already belongs to', async (t) => { encryptionKeys, }) - t.is(response1, MapeoRPC.InviteResponse.ACCEPT) + t.is(response1, LocalPeers.InviteResponse.ACCEPT) const response2 = await r1.invite(peers[0].id, { projectKey, encryptionKeys, }) - t.is(response2, MapeoRPC.InviteResponse.ALREADY) + t.is(response2, LocalPeers.InviteResponse.ALREADY) }) let inviteReceivedEventCount = 0 @@ -286,7 +286,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { }) test('trying to accept or reject non-existent invite throws', async (t) => { - const rpc = new MapeoRPC() + const rpc = new LocalPeers() const inviteApi = new InviteApi({ rpc, queries: { @@ -307,7 +307,7 @@ test('invitor disconnecting results in accept throwing', async (t) => { const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -345,7 +345,7 @@ test('invitor disconnecting results in invite reject response not throwing', asy const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -381,7 +381,7 @@ test('invitor disconnecting results in invite already response not throwing', as const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() let isMember = false @@ -422,7 +422,7 @@ test('addProject throwing results in invite accept throwing', async (t) => { const { rpc: r1, projectKey, encryptionKeys } = setup() - const r2 = new MapeoRPC() + const r2 = new LocalPeers() const inviteApi = new InviteApi({ rpc: r2, @@ -455,7 +455,7 @@ test('Invite from multiple peers', async (t) => { t.plan(5 + invitorCount) const { projectKey, encryptionKeys } = setup() - const invitee = new MapeoRPC() + const invitee = new LocalPeers() const inviteeKeyPair = NoiseSecretStream.keyPair() const projects = new Map() @@ -492,7 +492,7 @@ test('Invite from multiple peers', async (t) => { }) for (let i = 0; i < invitorCount; i++) { - const invitor = new MapeoRPC() + const invitor = new LocalPeers() const keyPair = NoiseSecretStream.keyPair() invitor.on('peers', async (peers) => { if (++connected === invitorCount) deferred.resolve() @@ -502,9 +502,9 @@ test('Invite from multiple peers', async (t) => { }) if (first === keyPair.publicKey.toString('hex')) { t.pass('One invitor did receive accept response') - t.is(response, MapeoRPC.InviteResponse.ACCEPT, 'accept response') + t.is(response, LocalPeers.InviteResponse.ACCEPT, 'accept response') } else { - t.is(response, MapeoRPC.InviteResponse.ALREADY, 'already response') + t.is(response, LocalPeers.InviteResponse.ALREADY, 'already response') } }) replicate(invitee, invitor, { kp1: inviteeKeyPair, kp2: keyPair }) @@ -517,7 +517,7 @@ test.skip('Invite from multiple peers, first disconnects before accepted, receiv t.plan(8 + invitorCount) const { projectKey, encryptionKeys } = setup() - const invitee = new MapeoRPC() + const invitee = new LocalPeers() const inviteeKeyPair = NoiseSecretStream.keyPair() const projects = new Map() @@ -562,7 +562,7 @@ test.skip('Invite from multiple peers, first disconnects before accepted, receiv }) for (let i = 0; i < invitorCount; i++) { - const invitor = new MapeoRPC() + const invitor = new LocalPeers() const keyPair = NoiseSecretStream.keyPair() const invitorId = keyPair.publicKey.toString('hex') invitor.on('peers', async (peers) => { @@ -575,9 +575,9 @@ test.skip('Invite from multiple peers, first disconnects before accepted, receiv }) if (invitorId === invitesReceived[1]) { t.pass('One invitor did receive accept response') - t.is(response, MapeoRPC.InviteResponse.ACCEPT, 'accept response') + t.is(response, LocalPeers.InviteResponse.ACCEPT, 'accept response') } else { - t.is(response, MapeoRPC.InviteResponse.ALREADY, 'already response') + t.is(response, LocalPeers.InviteResponse.ALREADY, 'already response') } } catch (e) { t.is( @@ -598,7 +598,7 @@ test.skip('Invite from multiple peers, first disconnects before accepted, receiv function setup() { const encryptionKeys = { auth: randomBytes(32) } const projectKey = KeyManager.generateProjectKeypair().publicKey - const rpc = new MapeoRPC() + const rpc = new LocalPeers() return { rpc, diff --git a/tests/rpc.js b/tests/rpc.js index 83cd90698..680dc0427 100644 --- a/tests/rpc.js +++ b/tests/rpc.js @@ -1,7 +1,7 @@ // @ts-check import test from 'brittle' import { - MapeoRPC, + LocalPeers, PeerDisconnectedError, TimeoutError, UnknownPeerError, @@ -15,8 +15,8 @@ import NoiseSecretStream from '@hyperswarm/secret-stream' test('Send invite and accept', async (t) => { t.plan(3) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -26,14 +26,14 @@ test('Send invite and accept', async (t) => { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) - t.is(response, MapeoRPC.InviteResponse.ACCEPT) + t.is(response, LocalPeers.InviteResponse.ACCEPT) }) r2.on('invite', (peerId, invite) => { t.ok(invite.projectKey.equals(projectKey), 'invite project key correct') r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) @@ -41,8 +41,8 @@ test('Send invite and accept', async (t) => { }) test('Send invite immediately', async (t) => { - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -62,16 +62,16 @@ test('Send invite immediately', async (t) => { r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) - t.is(await responsePromise, MapeoRPC.InviteResponse.ACCEPT) + t.is(await responsePromise, LocalPeers.InviteResponse.ACCEPT) }) test('Send invite and reject', async (t) => { t.plan(3) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -81,14 +81,14 @@ test('Send invite and reject', async (t) => { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) - t.is(response, MapeoRPC.InviteResponse.REJECT) + t.is(response, LocalPeers.InviteResponse.REJECT) }) r2.on('invite', (peerId, invite) => { t.ok(invite.projectKey.equals(projectKey), 'invite project key correct') r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.REJECT, + decision: LocalPeers.InviteResponse.REJECT, }) }) @@ -96,8 +96,8 @@ test('Send invite and reject', async (t) => { }) test('Invite to unknown peer', async (t) => { - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) const unknownPeerId = Buffer.allocUnsafe(32).fill(1).toString('hex') @@ -115,7 +115,7 @@ test('Invite to unknown peer', async (t) => { () => r2.inviteResponse(unknownPeerId, { projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }), UnknownPeerError ) @@ -123,8 +123,8 @@ test('Invite to unknown peer', async (t) => { test('Send invite and already on project', async (t) => { t.plan(3) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -134,14 +134,14 @@ test('Send invite and already on project', async (t) => { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) - t.is(response, MapeoRPC.InviteResponse.ALREADY) + t.is(response, LocalPeers.InviteResponse.ALREADY) }) r2.on('invite', (peerId, invite) => { t.ok(invite.projectKey.equals(projectKey), 'invite project key correct') r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ALREADY, + decision: LocalPeers.InviteResponse.ALREADY, }) }) @@ -150,8 +150,8 @@ test('Send invite and already on project', async (t) => { test('Send invite with encryption key', async (t) => { t.plan(4) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) const encryptionKeys = { @@ -165,7 +165,7 @@ test('Send invite with encryption key', async (t) => { projectKey, encryptionKeys, }) - t.is(response, MapeoRPC.InviteResponse.ACCEPT) + t.is(response, LocalPeers.InviteResponse.ACCEPT) }) r2.on('invite', (peerId, invite) => { @@ -177,7 +177,7 @@ test('Send invite with encryption key', async (t) => { ) r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) @@ -186,8 +186,8 @@ test('Send invite with encryption key', async (t) => { test('Send invite with project info', async (t) => { t.plan(4) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) const projectInfo = { name: 'MyProject' } @@ -199,7 +199,7 @@ test('Send invite with project info', async (t) => { projectInfo, encryptionKeys: { auth: randomBytes(32) }, }) - t.is(response, MapeoRPC.InviteResponse.ACCEPT) + t.is(response, LocalPeers.InviteResponse.ACCEPT) }) r2.on('invite', (peerId, invite) => { @@ -207,7 +207,7 @@ test('Send invite with project info', async (t) => { t.alike(invite.projectInfo, projectInfo, 'project info is sent with invite') r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) @@ -216,8 +216,8 @@ test('Send invite with project info', async (t) => { test('Disconnected peer shows in state', async (t) => { t.plan(6) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() let peerStateUpdates = 0 r1.on('peers', async (peers) => { @@ -237,8 +237,8 @@ test('Disconnected peer shows in state', async (t) => { test('Disconnect results in rejected invite', async (t) => { t.plan(2) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -268,9 +268,9 @@ test('Disconnect results in rejected invite', async (t) => { test('Invite to multiple peers', async (t) => { // This is catches not tracking invites by peer t.plan(2) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() - const r3 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() + const r3 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -287,7 +287,7 @@ test('Invite to multiple peers', async (t) => { ) t.alike( responses.sort(), - [MapeoRPC.InviteResponse.ACCEPT, MapeoRPC.InviteResponse.REJECT], + [LocalPeers.InviteResponse.ACCEPT, LocalPeers.InviteResponse.REJECT], 'One peer accepted, one rejected' ) }) @@ -295,14 +295,14 @@ test('Invite to multiple peers', async (t) => { r2.on('invite', (peerId, invite) => { r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) r3.on('invite', (peerId, invite) => { r3.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.REJECT, + decision: LocalPeers.InviteResponse.REJECT, }) }) @@ -314,8 +314,8 @@ test('Invite to multiple peers', async (t) => { test('Multiple invites to a peer, only one response', async (t) => { t.plan(2) let count = 0 - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -334,7 +334,7 @@ test('Multiple invites to a peer, only one response', async (t) => { encryptionKeys: { auth: randomBytes(32) }, }), ]) - const expected = Array(3).fill(MapeoRPC.InviteResponse.ACCEPT) + const expected = Array(3).fill(LocalPeers.InviteResponse.ACCEPT) t.alike(responses, expected) }) @@ -344,7 +344,7 @@ test('Multiple invites to a peer, only one response', async (t) => { t.is(count, 3) r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) @@ -356,8 +356,8 @@ test('Default: invites do not timeout', async (t) => { t.teardown(() => clock.uninstall()) t.plan(1) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -381,8 +381,8 @@ test('Invite timeout', async (t) => { t.teardown(() => clock.uninstall()) t.plan(1) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -402,8 +402,8 @@ test('Invite timeout', async (t) => { }) test('Reconnect peer and send invite', async (t) => { - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() const projectKey = Buffer.allocUnsafe(32).fill(0) @@ -418,7 +418,7 @@ test('Reconnect peer and send invite', async (t) => { t.ok(invite.projectKey.equals(projectKey), 'invite project key correct') r2.inviteResponse(peerId, { projectKey: invite.projectKey, - decision: MapeoRPC.InviteResponse.ACCEPT, + decision: LocalPeers.InviteResponse.ACCEPT, }) }) @@ -430,21 +430,20 @@ test('Reconnect peer and send invite', async (t) => { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) - t.is(response, MapeoRPC.InviteResponse.ACCEPT) + t.is(response, LocalPeers.InviteResponse.ACCEPT) }) test('invalid stream', (t) => { - const r1 = new MapeoRPC() + const r1 = new LocalPeers() const regularStream = new Duplex() - // @ts-expect-error t.exception(() => r1.connect(regularStream), 'Invalid stream') }) test('Send device info', async (t) => { t.plan(3) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() /** @type {import('../src/generated/rpc.js').DeviceInfo} */ const expectedDeviceInfo = { name: 'mapeo' } @@ -463,8 +462,8 @@ test('Send device info', async (t) => { }) test('Send device info immediately', async (t) => { - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() /** @type {import('../src/generated/rpc.js').DeviceInfo} */ const expectedDeviceInfo = { name: 'mapeo' } @@ -484,8 +483,8 @@ test('Send device info immediately', async (t) => { test('Reconnect peer and send device info', async (t) => { t.plan(6) - const r1 = new MapeoRPC() - const r2 = new MapeoRPC() + const r1 = new LocalPeers() + const r2 = new LocalPeers() /** @type {import('../src/generated/rpc.js').DeviceInfo} */ const expectedDeviceInfo = { name: 'mapeo' } From e301c85ed133500d8f8e01ea97f9402992111413 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 16:39:16 +0900 Subject: [PATCH 03/69] Handle deviceInfo internally, id -> deviceId --- src/rpc/index.js | 67 ++++++++++++++++++++++++++++++------ test-e2e/manager-invite.js | 8 ++--- test-e2e/members.js | 2 +- tests/invite-api.js | 28 ++++++++-------- tests/rpc.js | 69 +++++++++++++++++++------------------- 5 files changed, 111 insertions(+), 63 deletions(-) diff --git a/src/rpc/index.js b/src/rpc/index.js index 29e2deac0..5caa9b72c 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -24,9 +24,18 @@ const MESSAGE_TYPES = { } const MESSAGES_MAX_ID = Math.max.apply(null, [...Object.values(MESSAGE_TYPES)]) -/** @typedef {Peer['info']} PeerInfoInternal */ -/** @typedef {Omit & { status: Exclude }} PeerInfo */ -/** @typedef {'connecting' | 'connected' | 'disconnected'} PeerState */ +/** + * @typedef {object} PeerInfoBase + * @property {string} deviceId + * @property {string | undefined} name + */ +/** @typedef {PeerInfoBase & { status: 'connecting' }} PeerInfoConnecting */ +/** @typedef {PeerInfoBase & { status: 'connected', connectedAt: number }} PeerInfoConnected */ +/** @typedef {PeerInfoBase & { status: 'disconnected', disconnectedAt: number }} PeerInfoDisconnected */ + +/** @typedef {PeerInfoConnecting | PeerInfoConnected | PeerInfoDisconnected} PeerInfoInternal */ +/** @typedef {PeerInfoConnected | PeerInfoDisconnected} PeerInfo */ +/** @typedef {PeerInfoInternal['status']} PeerState */ /** @typedef {import('type-fest').SetNonNullable} InviteWithKeys */ /** @@ -44,6 +53,10 @@ class Peer { #connected /** @type {Map>>} */ pendingInvites = new Map() + /** @type {string | undefined} */ + #name + #connectedAt = 0 + #disconnectedAt = 0 /** * @param {object} options @@ -55,10 +68,36 @@ class Peer { this.#channel = channel this.#connected = pDefer() } + /** @returns {PeerInfoInternal} */ get info() { - return { - status: this.#state, - id: keyToId(this.#publicKey), + const deviceId = keyToId(this.#publicKey) + switch (this.#state) { + case 'connecting': + return { + status: this.#state, + deviceId, + name: this.#name, + } + case 'connected': + return { + status: this.#state, + deviceId, + name: this.#name, + connectedAt: this.#connectedAt, + } + case 'disconnected': + return { + status: this.#state, + deviceId, + name: this.#name, + disconnectedAt: this.#disconnectedAt, + } + /* c8 ignore next 4 */ + default: { + /** @type {never} */ + const _exhaustiveCheck = this.#state + return _exhaustiveCheck + } } } /** @@ -75,12 +114,16 @@ class Peer { return // TODO: report error - this should not happen } this.#state = 'connected' + this.#connectedAt = Date.now() this.#connected.resolve() break case 'disconnect': /* c8 ignore next */ if (this.#state === 'disconnected') return this.#state = 'disconnected' + this.#disconnectedAt = Date.now() + // Can just resolve this rather than reject, because #assertConnected will throw the error + this.#connected.resolve() for (const pending of this.pendingInvites.values()) { for (const { reject } of pending) { reject(new PeerDisconnectedError()) @@ -111,6 +154,10 @@ class Peer { const messageType = MESSAGE_TYPES.DeviceInfo this.#channel.messages[messageType].send(buf) } + /** @param {DeviceInfo} deviceInfo */ + receiveDeviceInfo(deviceInfo) { + this.#name = deviceInfo.name + } async #assertConnected() { await this.#connected.promise if (this.#state === 'connected' && !this.#channel.closed) return @@ -120,13 +167,12 @@ class Peer { } /** - * @typedef {object} MapeoRPCEvents + * @typedef {object} LocalPeersEvents * @property {(peers: PeerInfo[]) => void} peers Emitted whenever the connection status of peers changes. An array of peerInfo objects with a peer id and the peer connection status * @property {(peerId: string, invite: InviteWithKeys) => void} invite Emitted when an invite is received - * @property {(deviceInfo: DeviceInfo & { deviceId: string }) => void} device-info Emitted when we receive device info for a device */ -/** @extends {TypedEmitter} */ +/** @extends {TypedEmitter} */ export class LocalPeers extends TypedEmitter { /** @type {Map} */ #peers = new Map() @@ -348,7 +394,8 @@ export class LocalPeers extends TypedEmitter { } case 'DeviceInfo': { const deviceInfo = DeviceInfo.decode(value) - this.emit('device-info', { ...deviceInfo, deviceId: peerId }) + peer.receiveDeviceInfo(deviceInfo) + this.#emitPeers() break } /* c8 ignore next 5 */ diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index 437ba4297..dc19b283f 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -25,7 +25,7 @@ test('member invite accepted', async (t) => { creator[kRPC].on('peers', async (peers) => { t.is(peers.length, 1) - const response = await creatorProject.$member.invite(peers[0].id, { + const response = await creatorProject.$member.invite(peers[0].deviceId, { roleId: MEMBER_ROLE_ID, }) @@ -52,7 +52,7 @@ test('member invite accepted', async (t) => { joiner[kRPC].on('peers', (peers) => { t.is(peers.length, 1) - expectedInvitorPeerId = peers[0].id + expectedInvitorPeerId = peers[0].deviceId }) joiner.invite.on('invite-received', async (invite) => { @@ -119,7 +119,7 @@ test('member invite rejected', async (t) => { creator[kRPC].on('peers', async (peers) => { t.is(peers.length, 1) - const response = await creatorProject.$member.invite(peers[0].id, { + const response = await creatorProject.$member.invite(peers[0].deviceId, { roleId: MEMBER_ROLE_ID, }) @@ -146,7 +146,7 @@ test('member invite rejected', async (t) => { joiner[kRPC].on('peers', (peers) => { t.is(peers.length, 1) - expectedInvitorPeerId = peers[0].id + expectedInvitorPeerId = peers[0].deviceId }) joiner.invite.on('invite-received', async (invite) => { diff --git a/test-e2e/members.js b/test-e2e/members.js index cbef3fef8..8fe068478 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -191,7 +191,7 @@ function setup() { }) manager[kRPC].on('peers', (peers) => { - const deviceId = peers[0].id + const deviceId = peers[0].deviceId project.$member .invite(deviceId, { roleId }) .then(() => deferred.resolve(deviceId)) diff --git a/tests/invite-api.js b/tests/invite-api.js index 19f058dbd..4ba0909fb 100644 --- a/tests/invite-api.js +++ b/tests/invite-api.js @@ -33,13 +33,13 @@ test('invite-received event has expected payload', async (t) => { r2.on('peers', (peers) => { t.is(peers.length, 1) - expectedInvitorPeerId = peers[0].id + expectedInvitorPeerId = peers[0].deviceId }) r1.on('peers', (peers) => { t.is(peers.length, 1) - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, projectInfo: { name: 'Mapeo' }, @@ -83,7 +83,7 @@ test('Accept invite', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -127,7 +127,7 @@ test('Reject invite', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -169,7 +169,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -213,7 +213,7 @@ test('Receiving invite for project that peer already belongs to', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -257,14 +257,14 @@ test('Receiving invite for project that peer already belongs to', async (t) => { }) r1.on('peers', async (peers) => { - const response1 = await r1.invite(peers[0].id, { + const response1 = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) t.is(response1, LocalPeers.InviteResponse.ACCEPT) - const response2 = await r1.invite(peers[0].id, { + const response2 = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -322,7 +322,7 @@ test('invitor disconnecting results in accept throwing', async (t) => { r1.on('peers', async (peers) => { if (peers.length !== 1 || peers[0].status === 'disconnected') return await t.exception(() => { - return r1.invite(peers[0].id, { + return r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -359,7 +359,7 @@ test('invitor disconnecting results in invite reject response not throwing', asy if (peers.length !== 1 || peers[0].status === 'disconnected') return await t.exception(() => { - return r1.invite(peers[0].id, { + return r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -399,7 +399,7 @@ test('invitor disconnecting results in invite already response not throwing', as if (peers.length !== 1 || peers[0].status === 'disconnected') return await t.exception(() => { - return r1.invite(peers[0].id, { + return r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -435,7 +435,7 @@ test('addProject throwing results in invite accept throwing', async (t) => { }) r1.on('peers', (peers) => { - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -496,7 +496,7 @@ test('Invite from multiple peers', async (t) => { const keyPair = NoiseSecretStream.keyPair() invitor.on('peers', async (peers) => { if (++connected === invitorCount) deferred.resolve() - const response = await invitor.invite(peers[0].id, { + const response = await invitor.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -569,7 +569,7 @@ test.skip('Invite from multiple peers, first disconnects before accepted, receiv if (peers[0].status !== 'connected') return if (++connected === invitorCount) deferred.resolve() try { - const response = await invitor.invite(peers[0].id, { + const response = await invitor.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) diff --git a/tests/rpc.js b/tests/rpc.js index 680dc0427..e14633686 100644 --- a/tests/rpc.js +++ b/tests/rpc.js @@ -22,7 +22,7 @@ test('Send invite and accept', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -77,7 +77,7 @@ test('Send invite and reject', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -130,7 +130,7 @@ test('Send invite and already on project', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -161,7 +161,7 @@ test('Send invite with encryption key', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys, }) @@ -194,7 +194,7 @@ test('Send invite with project info', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, projectInfo, encryptionKeys: { auth: randomBytes(32) }, @@ -244,7 +244,7 @@ test('Disconnect results in rejected invite', async (t) => { r1.on('peers', async (peers) => { if (peers[0].status === 'connected') { - const invite = r1.invite(peers[0].id, { + const invite = r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -279,7 +279,7 @@ test('Invite to multiple peers', async (t) => { t.pass('connected to two peers') const responses = await Promise.all( peers.map((peer) => - r1.invite(peer.id, { + r1.invite(peer.deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -321,15 +321,15 @@ test('Multiple invites to a peer, only one response', async (t) => { r1.on('peers', async (peers) => { const responses = await Promise.all([ - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }), - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }), - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }), @@ -362,7 +362,7 @@ test('Default: invites do not timeout', async (t) => { const projectKey = Buffer.allocUnsafe(32).fill(0) r1.once('peers', async (peers) => { - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }).then( @@ -388,7 +388,7 @@ test('Invite timeout', async (t) => { r1.once('peers', async (peers) => { t.exception( - r1.invite(peers[0].id, { + r1.invite(peers[0].deviceId, { projectKey, timeout: 5000, encryptionKeys: { auth: randomBytes(32) }, @@ -426,7 +426,7 @@ test('Reconnect peer and send invite', async (t) => { const [peers] = await once(r1, 'peers') t.is(r1.peers.length, 1) t.is(peers[0].status, 'connected') - const response = await r1.invite(peers[0].id, { + const response = await r1.invite(peers[0].deviceId, { projectKey, encryptionKeys: { auth: randomBytes(32) }, }) @@ -440,8 +440,6 @@ test('invalid stream', (t) => { }) test('Send device info', async (t) => { - t.plan(3) - const r1 = new LocalPeers() const r2 = new LocalPeers() @@ -450,15 +448,18 @@ test('Send device info', async (t) => { r1.on('peers', async (peers) => { t.is(peers.length, 1) - r1.sendDeviceInfo(peers[0].id, expectedDeviceInfo) - }) - - r2.on('device-info', ({ deviceId, ...deviceInfo }) => { - t.ok(deviceId) - t.alike(deviceInfo, expectedDeviceInfo) + r1.sendDeviceInfo(peers[0].deviceId, expectedDeviceInfo) }) replicate(r1, r2) + + await new Promise((res) => { + r2.on('peers', (peers) => { + if (!(peers.length === 1 && peers[0].name)) return + t.is(peers[0].name, expectedDeviceInfo.name) + res(true) + }) + }) }) test('Send device info immediately', async (t) => { @@ -475,14 +476,16 @@ test('Send device info immediately', async (t) => { r1.sendDeviceInfo(kp2.publicKey.toString('hex'), expectedDeviceInfo) - const [{ deviceId, ...deviceInfo }] = await once(r2, 'device-info') - t.ok(deviceId) - t.alike(deviceInfo, expectedDeviceInfo) + await new Promise((res) => { + r2.on('peers', (peers) => { + if (!(peers.length === 1 && peers[0].name)) return + t.is(peers[0].name, expectedDeviceInfo.name) + res(true) + }) + }) }) test('Reconnect peer and send device info', async (t) => { - t.plan(6) - const r1 = new LocalPeers() const r2 = new LocalPeers() @@ -496,16 +499,14 @@ test('Reconnect peer and send device info', async (t) => { t.is(r1.peers.length, 1) t.is(r1.peers[0].status, 'disconnected') - r2.on('device-info', ({ deviceId, ...deviceInfo }) => { - t.ok(deviceId) - t.alike(deviceInfo, expectedDeviceInfo) - }) - replicate(r1, r2) - const [peers] = await once(r1, 'peers') + const [r1peers] = await once(r1, 'peers') t.is(r1.peers.length, 1) - t.is(peers[0].status, 'connected') + t.is(r1peers[0].status, 'connected') + + r1.sendDeviceInfo(r1peers[0].deviceId, expectedDeviceInfo) - r1.sendDeviceInfo(peers[0].id, expectedDeviceInfo) + const [r2Peers] = await once(r2, 'peers') + t.is(r2Peers[0].name, expectedDeviceInfo.name) }) From ae371fd8f339ba8d343e41a51d7a7849a203a003 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 17:01:43 +0900 Subject: [PATCH 04/69] Tests for stream error handling --- src/rpc/index.js | 3 +++ tests/helpers/rpc.js | 8 ++++---- tests/rpc.js | 15 +++++++++++++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/src/rpc/index.js b/src/rpc/index.js index 5caa9b72c..e1a019960 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -277,6 +277,9 @@ export class LocalPeers extends TypedEmitter { : Protomux.from(stream) this.#opening.add(stream.opened) + // No need to connect error handler to stream because Protomux does this, + // and errors are eventually handled by #closePeer + // noiseSecretStream.remotePublicKey can be null before the stream has // opened, so this helped awaits the open openedNoiseSecretStream(stream).then((stream) => { diff --git a/tests/helpers/rpc.js b/tests/helpers/rpc.js index c735096e5..0b71b0bfa 100644 --- a/tests/helpers/rpc.js +++ b/tests/helpers/rpc.js @@ -8,7 +8,6 @@ import NoiseSecretStream from '@hyperswarm/secret-stream' * @param {import('../../src/rpc/index.js').LocalPeers} rpc1 * @param {import('../../src/rpc/index.js').LocalPeers} rpc2 * @param { {kp1?: KeyPair, kp2?: KeyPair} } [keyPairs] - * @returns {() => Promise<[void, void]>} */ export function replicate( rpc1, @@ -32,20 +31,21 @@ export function replicate( rpc1.connect(n1) rpc2.connect(n2) - return async function destroy() { + /** @param {Error} [e] */ + return async function destroy(e) { return Promise.all([ /** @type {Promise} */ ( new Promise((res) => { n1.on('close', res) - n1.destroy() + n1.destroy(e) }) ), /** @type {Promise} */ ( new Promise((res) => { n2.on('close', res) - n2.destroy() + n2.destroy(e) }) ), ]) diff --git a/tests/rpc.js b/tests/rpc.js index e14633686..695d8a77a 100644 --- a/tests/rpc.js +++ b/tests/rpc.js @@ -225,7 +225,7 @@ test('Disconnected peer shows in state', async (t) => { if (peers[0].status === 'connected') { t.pass('peer appeared as connected') t.is(++peerStateUpdates, 1) - destroy() + destroy(new Error()) } else { t.pass('peer appeared as disconnected') t.is(++peerStateUpdates, 2) @@ -235,6 +235,16 @@ test('Disconnected peer shows in state', async (t) => { const destroy = replicate(r1, r2) }) +test('next tick disconnect does not throw', async (t) => { + const r1 = new LocalPeers() + const r2 = new LocalPeers() + + const destroy = replicate(r1, r2) + await Promise.resolve() + destroy(new Error()) + t.pass() +}) + test('Disconnect results in rejected invite', async (t) => { t.plan(2) const r1 = new LocalPeers() @@ -390,11 +400,12 @@ test('Invite timeout', async (t) => { t.exception( r1.invite(peers[0].deviceId, { projectKey, - timeout: 5000, + timeout: 1000, encryptionKeys: { auth: randomBytes(32) }, }), TimeoutError ) + // Not working right now, because of the new async code clock.tick(5001) }) From ccdf39f2519596e04dac31c49ff630bf052ced6e Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 17:04:19 +0900 Subject: [PATCH 05/69] remove unnecessary constructor --- src/rpc/index.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/rpc/index.js b/src/rpc/index.js index e1a019960..e1c6fb4c4 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -179,10 +179,6 @@ export class LocalPeers extends TypedEmitter { /** @type {Set>} */ #opening = new Set() - constructor() { - super() - } - static InviteResponse = InviteResponse_Decision /** From a52254b08f4bf00f1b34d5f93ef6d6d5fbc9f32c Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 17:10:54 +0900 Subject: [PATCH 06/69] return replication stream --- src/rpc/index.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rpc/index.js b/src/rpc/index.js index e1c6fb4c4..8ec334fc0 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -263,7 +263,8 @@ export class LocalPeers extends TypedEmitter { /** * Connect to a peer over an existing NoiseSecretStream * - * @param {import('../types.js').NoiseStream | import('../types.js').ProtocolStream} stream a NoiseSecretStream from @hyperswarm/secret-stream + * @param {import('../types.js').NoiseStream} stream a NoiseSecretStream from @hyperswarm/secret-stream + * @returns {import('../types.js').ReplicationStream} */ connect(stream) { if (!stream.noiseStream) throw new Error('Invalid stream') @@ -271,6 +272,7 @@ export class LocalPeers extends TypedEmitter { stream.userData && Protomux.isProtomux(stream.userData) ? stream.userData : Protomux.from(stream) + stream.userData = protomux this.#opening.add(stream.opened) // No need to connect error handler to stream because Protomux does this, @@ -315,7 +317,7 @@ export class LocalPeers extends TypedEmitter { // Do not emit peers now - will emit when connected }) - return stream + return stream.rawStream } /** @param {Buffer} publicKey */ From 50698e6935a675ddce8136dd7f07303de0ebecb3 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 17:34:32 +0900 Subject: [PATCH 07/69] Attach protomux instance to peer info --- src/rpc/index.js | 82 +++++++++++++++++++++++------------------------- tests/rpc.js | 10 ++++++ 2 files changed, 50 insertions(+), 42 deletions(-) diff --git a/src/rpc/index.js b/src/rpc/index.js index 8ec334fc0..273fa0ec6 100644 --- a/src/rpc/index.js +++ b/src/rpc/index.js @@ -30,7 +30,7 @@ const MESSAGES_MAX_ID = Math.max.apply(null, [...Object.values(MESSAGE_TYPES)]) * @property {string | undefined} name */ /** @typedef {PeerInfoBase & { status: 'connecting' }} PeerInfoConnecting */ -/** @typedef {PeerInfoBase & { status: 'connected', connectedAt: number }} PeerInfoConnected */ +/** @typedef {PeerInfoBase & { status: 'connected', connectedAt: number, protomux: Protomux }} PeerInfoConnected */ /** @typedef {PeerInfoBase & { status: 'disconnected', disconnectedAt: number }} PeerInfoDisconnected */ /** @typedef {PeerInfoConnecting | PeerInfoConnected | PeerInfoDisconnected} PeerInfoInternal */ @@ -57,6 +57,8 @@ class Peer { #name #connectedAt = 0 #disconnectedAt = 0 + /** @type {Protomux} */ + #protomux /** * @param {object} options @@ -84,6 +86,7 @@ class Peer { deviceId, name: this.#name, connectedAt: this.#connectedAt, + protomux: this.#protomux, } case 'disconnected': return { @@ -100,38 +103,32 @@ class Peer { } } } - /** - * Poor-man's finite state machine. Rather than a `setState` method, only - * allows specific transitions between states. - * - * @param {'connect' | 'disconnect'} type - */ - action(type) { - switch (type) { - case 'connect': - /* c8 ignore next 3 */ - if (this.#state !== 'connecting') { - return // TODO: report error - this should not happen - } - this.#state = 'connected' - this.#connectedAt = Date.now() - this.#connected.resolve() - break - case 'disconnect': - /* c8 ignore next */ - if (this.#state === 'disconnected') return - this.#state = 'disconnected' - this.#disconnectedAt = Date.now() - // Can just resolve this rather than reject, because #assertConnected will throw the error - this.#connected.resolve() - for (const pending of this.pendingInvites.values()) { - for (const { reject } of pending) { - reject(new PeerDisconnectedError()) - } - } - this.pendingInvites.clear() - break + /** @param {Protomux} protomux */ + connect(protomux) { + this.#protomux = protomux + /* c8 ignore next 3 */ + if (this.#state !== 'connecting') { + return // TODO: report error - this should not happen } + this.#state = 'connected' + this.#connectedAt = Date.now() + this.#connected.resolve() + } + disconnect() { + // @ts-ignore - easier to ignore this than handle this for TS - avoids holding a reference to old Protomux instances + this.#protomux = undefined + /* c8 ignore next */ + if (this.#state === 'disconnected') return + this.#state = 'disconnected' + this.#disconnectedAt = Date.now() + // Can just resolve this rather than reject, because #assertConnected will throw the error + this.#connected.resolve() + for (const pending of this.pendingInvites.values()) { + for (const { reject } of pending) { + reject(new PeerDisconnectedError()) + } + } + this.pendingInvites.clear() } /** @param {InviteWithKeys} invite */ async sendInvite(invite) { @@ -301,7 +298,7 @@ export class LocalPeers extends TypedEmitter { userData: null, protocol: PROTOCOL_NAME, messages, - onopen: this.#openPeer.bind(this, remotePublicKey), + onopen: this.#openPeer.bind(this, remotePublicKey, protomux), onclose: this.#closePeer.bind(this, remotePublicKey), }) channel.open() @@ -310,7 +307,7 @@ export class LocalPeers extends TypedEmitter { const existingPeer = this.#peers.get(peerId) /* c8 ignore next 3 */ if (existingPeer && existingPeer.info.status !== 'disconnected') { - existingPeer.action('disconnect') // Should not happen, but in case + existingPeer.disconnect() // Should not happen, but in case } const peer = new Peer({ publicKey: remotePublicKey, channel }) this.#peers.set(peerId, peer) @@ -320,17 +317,18 @@ export class LocalPeers extends TypedEmitter { return stream.rawStream } - /** @param {Buffer} publicKey */ - #openPeer(publicKey) { + /** + * @param {Buffer} publicKey + * @param {Protomux} protomux + */ + #openPeer(publicKey, protomux) { const peerId = keyToId(publicKey) const peer = this.#peers.get(peerId) /* c8 ignore next */ if (!peer) return // TODO: report error - this should not happen - // No-op if no change in state - /* c8 ignore next */ - if (peer.info.status === 'connected') return // TODO: report error - this should not happen - peer.action('connect') - this.#emitPeers() + const wasConnected = peer.info.status === 'connected' + peer.connect(protomux) + if (!wasConnected) this.#emitPeers() } /** @param {Buffer} publicKey */ @@ -343,7 +341,7 @@ export class LocalPeers extends TypedEmitter { /* c8 ignore next */ if (peer.info.status === 'disconnected') return // TODO: Track reasons for closing - peer.action('disconnect') + peer.disconnect() this.#emitPeers() } diff --git a/tests/rpc.js b/tests/rpc.js index 695d8a77a..bb1cc46e4 100644 --- a/tests/rpc.js +++ b/tests/rpc.js @@ -12,6 +12,7 @@ import { Duplex } from 'streamx' import { replicate } from './helpers/rpc.js' import { randomBytes } from 'node:crypto' import NoiseSecretStream from '@hyperswarm/secret-stream' +import Protomux from 'protomux' test('Send invite and accept', async (t) => { t.plan(3) @@ -521,3 +522,12 @@ test('Reconnect peer and send device info', async (t) => { const [r2Peers] = await once(r2, 'peers') t.is(r2Peers[0].name, expectedDeviceInfo.name) }) + +test('connected peer has protomux instance', async (t) => { + const r1 = new LocalPeers() + const r2 = new LocalPeers() + replicate(r1, r2) + const [[peer]] = await once(r1, 'peers') + t.is(peer.status, 'connected') + t.ok(Protomux.isProtomux(peer.protomux)) +}) From ae35e9ca322268af8ee4660ea3b457d43d717787 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 17:47:02 +0900 Subject: [PATCH 08/69] rename and re-organize --- src/invite-api.js | 2 +- src/{rpc/index.js => local-peers.js} | 12 ++++++------ src/mapeo-manager.js | 2 +- src/mapeo-project.js | 2 +- src/member-api.js | 2 +- test-e2e/manager-invite.js | 2 +- test-e2e/members.js | 2 +- test-types/data-types.ts | 2 +- tests/helpers/{rpc.js => local-peers.js} | 4 ++-- tests/invite-api.js | 4 ++-- tests/{rpc.js => local-peers.js} | 4 ++-- 11 files changed, 19 insertions(+), 19 deletions(-) rename src/{rpc/index.js => local-peers.js} (96%) rename tests/helpers/{rpc.js => local-peers.js} (90%) rename tests/{rpc.js => local-peers.js} (99%) diff --git a/src/invite-api.js b/src/invite-api.js index 630d1f42b..03dd72970 100644 --- a/src/invite-api.js +++ b/src/invite-api.js @@ -47,7 +47,7 @@ export class InviteApi extends TypedEmitter { /** * @param {Object} options - * @param {import('./rpc/index.js').LocalPeers} options.rpc + * @param {import('./local-peers.js').LocalPeers} options.rpc * @param {object} options.queries * @param {(projectId: string) => Promise} options.queries.isMember * @param {(invite: import('./generated/rpc.js').Invite) => Promise} options.queries.addProject diff --git a/src/rpc/index.js b/src/local-peers.js similarity index 96% rename from src/rpc/index.js rename to src/local-peers.js index 273fa0ec6..5989a5bd8 100644 --- a/src/rpc/index.js +++ b/src/local-peers.js @@ -1,14 +1,14 @@ // @ts-check import { TypedEmitter } from 'tiny-typed-emitter' import Protomux from 'protomux' -import { openedNoiseSecretStream, keyToId } from '../utils.js' +import { openedNoiseSecretStream, keyToId } from './utils.js' import cenc from 'compact-encoding' import { DeviceInfo, Invite, InviteResponse, InviteResponse_Decision, -} from '../generated/rpc.js' +} from './generated/rpc.js' import pDefer from 'p-defer' const PROTOCOL_NAME = 'mapeo/rpc' @@ -16,7 +16,7 @@ const PROTOCOL_NAME = 'mapeo/rpc' // Protomux message types depend on the order that messages are added to a // channel (this needs to remain consistent). To avoid breaking changes, the // types here should not change. -/** @satisfies {{ [k in keyof typeof import('../generated/rpc.js')]?: number }} */ +/** @satisfies {{ [k in keyof typeof import('./generated/rpc.js')]?: number }} */ const MESSAGE_TYPES = { Invite: 0, InviteResponse: 1, @@ -36,7 +36,7 @@ const MESSAGES_MAX_ID = Math.max.apply(null, [...Object.values(MESSAGE_TYPES)]) /** @typedef {PeerInfoConnecting | PeerInfoConnected | PeerInfoDisconnected} PeerInfoInternal */ /** @typedef {PeerInfoConnected | PeerInfoDisconnected} PeerInfo */ /** @typedef {PeerInfoInternal['status']} PeerState */ -/** @typedef {import('type-fest').SetNonNullable} InviteWithKeys */ +/** @typedef {import('type-fest').SetNonNullable} InviteWithKeys */ /** * @template ValueType @@ -260,8 +260,8 @@ export class LocalPeers extends TypedEmitter { /** * Connect to a peer over an existing NoiseSecretStream * - * @param {import('../types.js').NoiseStream} stream a NoiseSecretStream from @hyperswarm/secret-stream - * @returns {import('../types.js').ReplicationStream} + * @param {import('./types.js').NoiseStream} stream a NoiseSecretStream from @hyperswarm/secret-stream + * @returns {import('./types.js').ReplicationStream} */ connect(stream) { if (!stream.noiseStream) throw new Error('Invalid stream') diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index b44128b8e..6573c0385 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -23,7 +23,7 @@ import { projectKeyToPublicId, } from './utils.js' import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' -import { LocalPeers } from './rpc/index.js' +import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' import { LocalDiscovery } from './discovery/local-discovery.js' diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 8d6af591a..ed24cbc50 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -67,7 +67,7 @@ export class MapeoProject { * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.sharedDb * @param {IndexWriter} opts.sharedIndexWriter * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data - * @param {import('./rpc/index.js').LocalPeers} opts.rpc + * @param {import('./local-peers.js').LocalPeers} opts.rpc * */ constructor({ diff --git a/src/member-api.js b/src/member-api.js index 6f92659db..1d2f490d2 100644 --- a/src/member-api.js +++ b/src/member-api.js @@ -22,7 +22,7 @@ export class MemberApi extends TypedEmitter { * @param {import('./core-ownership.js').CoreOwnership} opts.coreOwnership * @param {import('./generated/keys.js').EncryptionKeys} opts.encryptionKeys * @param {Buffer} opts.projectKey - * @param {import('./rpc/index.js').LocalPeers} opts.rpc + * @param {import('./local-peers.js').LocalPeers} opts.rpc * @param {Object} opts.dataTypes * @param {Pick} opts.dataTypes.deviceInfo * @param {Pick} opts.dataTypes.project diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index dc19b283f..4a0b866ce 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -5,7 +5,7 @@ import RAM from 'random-access-memory' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' import { MapeoManager, kRPC } from '../src/mapeo-manager.js' -import { replicate } from '../tests/helpers/rpc.js' +import { replicate } from '../tests/helpers/local-peers.js' test('member invite accepted', async (t) => { t.plan(10) diff --git a/test-e2e/members.js b/test-e2e/members.js index 8fe068478..64c28dee9 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -11,7 +11,7 @@ import { MEMBER_ROLE_ID, NO_ROLE_CAPABILITIES, } from '../src/capabilities.js' -import { replicate } from '../tests/helpers/rpc.js' +import { replicate } from '../tests/helpers/local-peers.js' test('getting yourself after creating project', async (t) => { const { manager } = setup() diff --git a/test-types/data-types.ts b/test-types/data-types.ts index 74cef62a9..10b2fcb15 100644 --- a/test-types/data-types.ts +++ b/test-types/data-types.ts @@ -14,7 +14,7 @@ import { drizzle } from 'drizzle-orm/better-sqlite3' import RAM from 'random-access-memory' import { IndexWriter } from '../dist/index-writer/index.js' import { projectSettingsTable } from '../dist/schema/client.js' -import { LocalPeers } from '../dist/rpc/index.js' +import { LocalPeers } from '../dist/local-peers.js' import { Expect, type Equal } from './utils.js' type Forks = { forks: string[] } diff --git a/tests/helpers/rpc.js b/tests/helpers/local-peers.js similarity index 90% rename from tests/helpers/rpc.js rename to tests/helpers/local-peers.js index 0b71b0bfa..1b7a8ce84 100644 --- a/tests/helpers/rpc.js +++ b/tests/helpers/local-peers.js @@ -5,8 +5,8 @@ import NoiseSecretStream from '@hyperswarm/secret-stream' */ /** - * @param {import('../../src/rpc/index.js').LocalPeers} rpc1 - * @param {import('../../src/rpc/index.js').LocalPeers} rpc2 + * @param {import('../../src/local-peers.js').LocalPeers} rpc1 + * @param {import('../../src/local-peers.js').LocalPeers} rpc2 * @param { {kp1?: KeyPair, kp2?: KeyPair} } [keyPairs] */ export function replicate( diff --git a/tests/invite-api.js b/tests/invite-api.js index 4ba0909fb..010a955fc 100644 --- a/tests/invite-api.js +++ b/tests/invite-api.js @@ -1,10 +1,10 @@ import test from 'brittle' import { randomBytes } from 'crypto' import { KeyManager } from '@mapeo/crypto' -import { LocalPeers } from '../src/rpc/index.js' +import { LocalPeers } from '../src/local-peers.js' import { InviteApi } from '../src/invite-api.js' import { projectKeyToPublicId } from '../src/utils.js' -import { replicate } from './helpers/rpc.js' +import { replicate } from './helpers/local-peers.js' import NoiseSecretStream from '@hyperswarm/secret-stream' import pDefer from 'p-defer' diff --git a/tests/rpc.js b/tests/local-peers.js similarity index 99% rename from tests/rpc.js rename to tests/local-peers.js index bb1cc46e4..dc704f8a4 100644 --- a/tests/rpc.js +++ b/tests/local-peers.js @@ -5,11 +5,11 @@ import { PeerDisconnectedError, TimeoutError, UnknownPeerError, -} from '../src/rpc/index.js' +} from '../src/local-peers.js' import FakeTimers from '@sinonjs/fake-timers' import { once } from 'events' import { Duplex } from 'streamx' -import { replicate } from './helpers/rpc.js' +import { replicate } from './helpers/local-peers.js' import { randomBytes } from 'node:crypto' import NoiseSecretStream from '@hyperswarm/secret-stream' import Protomux from 'protomux' From be64a3d8ff714d596b72a71379a281a93c03231c Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 20:19:09 +0900 Subject: [PATCH 09/69] revert changes outside scope of PR --- src/mapeo-manager.js | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 6573c0385..32965a237 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -17,7 +17,6 @@ import { ProjectKeys } from './generated/keys.js' import { deNullify, getDeviceId, - keyToId, projectIdToNonce, projectKeyToId, projectKeyToPublicId, @@ -25,7 +24,6 @@ import { import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' -import { LocalDiscovery } from './discovery/local-discovery.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -52,7 +50,6 @@ export class MapeoManager { #deviceId #rpc #invite - #localDiscovery /** * @param {Object} opts @@ -106,17 +103,6 @@ export class MapeoManager { } else { this.#coreStorage = coreStorage } - - this.#localDiscovery = new LocalDiscovery({ - identityKeypair: this.#keyManager.getIdentityKeypair(), - }) - - this.#localDiscovery.on('connection', (connection) => { - this.#handleDiscoveryConnection(connection).catch((e) => { - // Ignore errors here for now - console.error('Error handling discovery connection', e) - }) - }) } /** @@ -396,18 +382,6 @@ export class MapeoManager { return projectPublicId } - /** - * @param {import('./discovery/local-discovery.js').OpenedNoiseStream} connection - */ - async #handleDiscoveryConnection(connection) { - const peerId = keyToId(connection.remotePublicKey) - this.#rpc.connect(connection) - const { name } = await this.getDeviceInfo() - if (name) { - this.#rpc.sendDeviceInfo(peerId, { name }) - } - } - /** * @template {import('type-fest').Exact} T * @param {T} deviceInfo From ee2020ea11ec993e1e9e62615e2c2ec05a990e87 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 21:15:26 +0900 Subject: [PATCH 10/69] WIP initial work --- src/mapeo-manager.js | 44 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 32965a237..e76a384f2 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -17,6 +17,8 @@ import { ProjectKeys } from './generated/keys.js' import { deNullify, getDeviceId, + keyToId, + openedNoiseSecretStream, projectIdToNonce, projectKeyToId, projectKeyToPublicId, @@ -24,6 +26,7 @@ import { import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' +import { LocalDiscovery } from './discovery/local-discovery.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -48,8 +51,9 @@ export class MapeoManager { #coreStorage #dbFolder #deviceId - #rpc + #localPeers #invite + #localDiscovery /** * @param {Object} opts @@ -69,7 +73,7 @@ export class MapeoManager { migrationsFolder: new URL('../drizzle/client', import.meta.url).pathname, }) - this.#rpc = new LocalPeers() + this.#localPeers = new LocalPeers() this.#keyManager = new KeyManager(rootKey) this.#deviceId = getDeviceId(this.#keyManager) this.#projectSettingsIndexWriter = new IndexWriter({ @@ -79,7 +83,7 @@ export class MapeoManager { this.#activeProjects = new Map() this.#invite = new InviteApi({ - rpc: this.#rpc, + rpc: this.#localPeers, queries: { isMember: async (projectId) => { const projectExists = this.#db @@ -99,17 +103,43 @@ export class MapeoManager { if (typeof coreStorage === 'string') { const pool = new RandomAccessFilePool(MAX_FILE_DESCRIPTORS) // @ts-ignore - this.#coreStorage = Hypercore.createStorage(coreStorage, { pool }) + this.#coreStorage = Hypercore.defaultStorage(coreStorage, { pool }) } else { this.#coreStorage = coreStorage } + + this.#localDiscovery = new LocalDiscovery({ + identityKeypair: this.#keyManager.getIdentityKeypair(), + }) + this.#localDiscovery.on('connection', this.replicate.bind(this)) } /** * MapeoRPC instance, used for tests */ get [kRPC]() { - return this.#rpc + return this.#localPeers + } + + /** + * Replicate Mapeo to a `@hyperswarm/secret-stream`. Should only be used for + * local (trusted) connections, because the RPC channel key is public + * + * @param {import('@hyperswarm/secret-stream')} noiseStream + */ + replicate(noiseStream) { + const replicationStream = this.#localPeers.connect(noiseStream) + Promise.all([this.getDeviceInfo(), openedNoiseSecretStream(noiseStream)]) + .then(([{ name }, openedNoiseStream]) => { + if (openedNoiseStream.destroyed || !name) return + const peerId = keyToId(openedNoiseStream.remotePublicKey) + return this.#localPeers.sendDeviceInfo(peerId, { name }) + }) + .catch((e) => { + // Ignore error but log + console.error('Failed to send device info to peer', e) + }) + return replicationStream } /** @@ -213,7 +243,7 @@ export class MapeoManager { projectSecretKey: projectKeypair.secretKey, sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, - rpc: this.#rpc, + rpc: this.#localPeers, }) // 5. Write project name and any other relevant metadata to project instance @@ -269,7 +299,7 @@ export class MapeoManager { keyManager: this.#keyManager, sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, - rpc: this.#rpc, + rpc: this.#localPeers, }) // 3. Keep track of project instance as we know it's a properly existing project From 0cd25da5ec600334cb527397a63034e1c153a02c Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 26 Oct 2023 23:05:23 +0900 Subject: [PATCH 11/69] Tie everything together --- src/local-peers.js | 21 +++++++++++++++------ src/mapeo-manager.js | 27 ++++++++++++++------------- src/mapeo-project.js | 34 +++++++++++++++++++++++++++++----- src/sync/sync-controller.js | 35 ++++------------------------------- test-types/data-types.ts | 2 +- 5 files changed, 63 insertions(+), 56 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index 5989a5bd8..32c0d2305 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -30,7 +30,7 @@ const MESSAGES_MAX_ID = Math.max.apply(null, [...Object.values(MESSAGE_TYPES)]) * @property {string | undefined} name */ /** @typedef {PeerInfoBase & { status: 'connecting' }} PeerInfoConnecting */ -/** @typedef {PeerInfoBase & { status: 'connected', connectedAt: number, protomux: Protomux }} PeerInfoConnected */ +/** @typedef {PeerInfoBase & { status: 'connected', connectedAt: number, protomux: Protomux }} PeerInfoConnected */ /** @typedef {PeerInfoBase & { status: 'disconnected', disconnectedAt: number }} PeerInfoDisconnected */ /** @typedef {PeerInfoConnecting | PeerInfoConnected | PeerInfoDisconnected} PeerInfoInternal */ @@ -57,7 +57,7 @@ class Peer { #name #connectedAt = 0 #disconnectedAt = 0 - /** @type {Protomux} */ + /** @type {Protomux} */ #protomux /** @@ -103,7 +103,7 @@ class Peer { } } } - /** @param {Protomux} protomux */ + /** @param {Protomux} protomux */ connect(protomux) { this.#protomux = protomux /* c8 ignore next 3 */ @@ -166,7 +166,9 @@ class Peer { /** * @typedef {object} LocalPeersEvents * @property {(peers: PeerInfo[]) => void} peers Emitted whenever the connection status of peers changes. An array of peerInfo objects with a peer id and the peer connection status + * @property {(peer: PeerInfoConnected) => void} peer-add Emitted when a new peer is connected * @property {(peerId: string, invite: InviteWithKeys) => void} invite Emitted when an invite is received + * @property {(discoveryKey: Buffer, stream: import('./types.js').ReplicationStream) => void} discovery-key Emitted when a new hypercore is replicated (by a peer) to a peer replication stream (passed as the second parameter) */ /** @extends {TypedEmitter} */ @@ -272,6 +274,13 @@ export class LocalPeers extends TypedEmitter { stream.userData = protomux this.#opening.add(stream.opened) + protomux.pair( + { protocol: 'hypercore/alpha' }, + /** @param {Buffer} discoveryKey */ async (discoveryKey) => { + this.emit('discovery-key', discoveryKey, stream.rawStream) + } + ) + // No need to connect error handler to stream because Protomux does this, // and errors are eventually handled by #closePeer @@ -319,16 +328,16 @@ export class LocalPeers extends TypedEmitter { /** * @param {Buffer} publicKey - * @param {Protomux} protomux + * @param {Protomux} protomux */ #openPeer(publicKey, protomux) { const peerId = keyToId(publicKey) const peer = this.#peers.get(peerId) /* c8 ignore next */ if (!peer) return // TODO: report error - this should not happen - const wasConnected = peer.info.status === 'connected' peer.connect(protomux) - if (!wasConnected) this.#emitPeers() + this.#emitPeers() + this.emit('peer-add', /** @type {PeerInfoConnected} */ (peer.info)) } /** @param {Buffer} publicKey */ diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index e76a384f2..90a448933 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -235,15 +235,10 @@ export class MapeoManager { }) // 4. Create MapeoProject instance - const project = new MapeoProject({ - ...this.#projectStorage(projectId), + const project = this.#getProjectInstance({ encryptionKeys, - keyManager: this.#keyManager, projectKey: projectKeypair.publicKey, projectSecretKey: projectKeypair.secretKey, - sharedDb: this.#db, - sharedIndexWriter: this.#projectSettingsIndexWriter, - rpc: this.#localPeers, }) // 5. Write project name and any other relevant metadata to project instance @@ -293,19 +288,25 @@ export class MapeoManager { projectId ) - const project = new MapeoProject({ + const project = this.#getProjectInstance(projectKeys) + + // 3. Keep track of project instance as we know it's a properly existing project + this.#activeProjects.set(projectPublicId, project) + + return project + } + + /** @param {ProjectKeys} projectKeys */ + #getProjectInstance(projectKeys) { + const projectId = keyToId(projectKeys.projectKey) + return new MapeoProject({ ...this.#projectStorage(projectId), ...projectKeys, keyManager: this.#keyManager, sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, - rpc: this.#localPeers, + localPeers: this.#localPeers, }) - - // 3. Keep track of project instance as we know it's a properly existing project - this.#activeProjects.set(projectPublicId, project) - - return project } /** diff --git a/src/mapeo-project.js b/src/mapeo-project.js index ed24cbc50..cf9805174 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -33,6 +33,7 @@ import { Capabilities } from './capabilities.js' import { getDeviceId, projectKeyToId, valueOf } from './utils.js' import { MemberApi } from './member-api.js' import { SyncController } from './sync/sync-controller.js' +import Hypercore from 'hypercore' /** @typedef {Omit} EditableProjectSettings */ @@ -67,7 +68,7 @@ export class MapeoProject { * @param {import('drizzle-orm/better-sqlite3').BetterSQLite3Database} opts.sharedDb * @param {IndexWriter} opts.sharedIndexWriter * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data - * @param {import('./local-peers.js').LocalPeers} opts.rpc + * @param {import('./local-peers.js').LocalPeers} opts.localPeers * */ constructor({ @@ -79,7 +80,7 @@ export class MapeoProject { projectKey, projectSecretKey, encryptionKeys, - rpc, + localPeers, }) { this.#deviceId = getDeviceId(keyManager) this.#projectId = projectKeyToId(projectKey) @@ -237,7 +238,7 @@ export class MapeoProject { // @ts-expect-error encryptionKeys, projectKey, - rpc, + rpc: localPeers, dataTypes: { deviceInfo: this.#dataTypes.deviceInfo, project: this.#dataTypes.projectSettings, @@ -249,6 +250,26 @@ export class MapeoProject { capabilities: this.#capabilities, }) + // Replicate already connected local peers + for (const peer of localPeers.peers) { + if (peer.status !== 'connected') continue + this.#syncController.replicate(peer.protomux) + } + + localPeers.on('discovery-key', (discoveryKey, stream) => { + // The core identified by this discovery key might not be part of this + // project, but we can't know that so we will request it from the peer if + // we don't have it. The peer will not return the core key unless it _is_ + // part of this project + this.#coreManager.handleDiscoveryKey(discoveryKey, stream) + }) + + // When a new peer is found, try to replicate (if it is not a member of the + // project it will fail the capability check and be ignored) + localPeers.on('peer-add', (peer) => { + this.#syncController.replicate(peer.protomux) + }) + ///////// 4. Write core ownership record const deferred = pDefer() @@ -396,11 +417,14 @@ export class MapeoProject { /** * - * @param {import('./types.js').ReplicationStream} stream + * @param {Exclude[0], boolean>} stream A duplex stream, a @hyperswarm/secret-stream, or a Protomux instance * @returns */ [kReplicate](stream) { - return this.#syncController.replicate(stream) + const replicationStream = Hypercore.createProtocolStream(stream, {}) + const protomux = replicationStream.noiseStream.userData + // @ts-ignore - got fed up jumping through hoops to keep TS heppy + return this.#syncController.replicate(protomux) } /** diff --git a/src/sync/sync-controller.js b/src/sync/sync-controller.js index f68d2f6e1..a32b6cb95 100644 --- a/src/sync/sync-controller.js +++ b/src/sync/sync-controller.js @@ -1,6 +1,4 @@ -import Hypercore from 'hypercore' import { TypedEmitter } from 'tiny-typed-emitter' -import Protomux from 'protomux' import { SyncState } from './sync-state.js' import { PeerSyncController } from './peer-sync-controller.js' @@ -8,7 +6,7 @@ export class SyncController extends TypedEmitter { #syncState #coreManager #capabilities - /** @type {Map} */ + /** @type {Map} */ #peerSyncControllers = new Map() /** @@ -30,35 +28,10 @@ export class SyncController extends TypedEmitter { } /** - * @param {Exclude[0], boolean>} stream A duplex stream, a @hyperswarm/secret-stream, or a Protomux instance + * @param {import('protomux')} protomux A protomux instance */ - replicate(stream) { - if ( - Protomux.isProtomux(stream) || - ('userData' in stream && Protomux.isProtomux(stream.userData)) || - ('noiseStream' in stream && - Protomux.isProtomux(stream.noiseStream.userData)) - ) { - console.warn( - 'Passed an existing protocol stream to syncController.replicate(). Currently any pairing for the `hypercore/alpha` protocol is overwritten' - ) - } - const protocolStream = Hypercore.createProtocolStream(stream, { - ondiscoverykey: /** @param {Buffer} discoveryKey */ (discoveryKey) => { - return this.#coreManager.handleDiscoveryKey(discoveryKey, stream) - }, - }) - const protomux = - // Need to coerce this until we update Hypercore.createProtocolStream types - /** @type {import('protomux')} */ ( - protocolStream.noiseStream.userData - ) - if (!protomux) throw new Error('Invalid stream') - - if (this.#peerSyncControllers.has(protomux)) { - console.warn('Already replicating to this stream') - return - } + replicate(protomux) { + if (this.#peerSyncControllers.has(protomux)) return const peerSyncController = new PeerSyncController({ protomux, diff --git a/test-types/data-types.ts b/test-types/data-types.ts index 10b2fcb15..aa24bed80 100644 --- a/test-types/data-types.ts +++ b/test-types/data-types.ts @@ -36,7 +36,7 @@ const mapeoProject = new MapeoProject({ tables: [projectSettingsTable], sqlite, }), - rpc: new LocalPeers(), + localPeers: new LocalPeers(), }) ///// Observations From 395345de266f30645267d119cb9d8ce2ba618171 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 27 Oct 2023 14:46:30 +0900 Subject: [PATCH 12/69] rename getProjectInstance --- src/mapeo-manager.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 90a448933..e37110e2e 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -235,7 +235,7 @@ export class MapeoManager { }) // 4. Create MapeoProject instance - const project = this.#getProjectInstance({ + const project = this.#createProjectInstance({ encryptionKeys, projectKey: projectKeypair.publicKey, projectSecretKey: projectKeypair.secretKey, @@ -288,7 +288,7 @@ export class MapeoManager { projectId ) - const project = this.#getProjectInstance(projectKeys) + const project = this.#createProjectInstance(projectKeys) // 3. Keep track of project instance as we know it's a properly existing project this.#activeProjects.set(projectPublicId, project) @@ -297,7 +297,7 @@ export class MapeoManager { } /** @param {ProjectKeys} projectKeys */ - #getProjectInstance(projectKeys) { + #createProjectInstance(projectKeys) { const projectId = keyToId(projectKeys.projectKey) return new MapeoProject({ ...this.#projectStorage(projectId), From 7963a3e2b14ab6a1a8b3cfaf4a2f14c9b8aa03a9 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 27 Oct 2023 11:47:50 +0900 Subject: [PATCH 13/69] feat: client.listLocalPeers() & `local-peers` evt --- src/mapeo-manager.js | 52 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index e37110e2e..999226620 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -27,6 +27,7 @@ import { RandomAccessFilePool } from './core-manager/random-access-file-pool.js' import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' import { LocalDiscovery } from './discovery/local-discovery.js' +import { TypedEmitter } from 'tiny-typed-emitter' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -40,7 +41,19 @@ const MAX_FILE_DESCRIPTORS = 768 export const kRPC = Symbol('rpc') -export class MapeoManager { +/** + * @typedef {Omit} PublicPeerInfo + */ + +/** + * @typedef {object} MapeoManagerEvents + * @property {(peers: PublicPeerInfo[]) => void} local-peers Emitted when the list of connected peers changes (new ones added, or connection status changes) + */ + +/** + * @extends {TypedEmitter} + */ +export class MapeoManager extends TypedEmitter { #keyManager #projectSettingsIndexWriter #db @@ -62,6 +75,7 @@ export class MapeoManager { * @param {string | import('./types.js').CoreStorage} opts.coreStorage Folder for hypercore storage or a function that returns a RandomAccessStorage instance */ constructor({ rootKey, dbFolder, coreStorage }) { + super() this.#dbFolder = dbFolder const sqlite = new Database( dbFolder === ':memory:' @@ -74,6 +88,10 @@ export class MapeoManager { }) this.#localPeers = new LocalPeers() + this.#localPeers.on('peers', (peers) => { + this.emit('local-peers', omitPeerProtomux(peers)) + }) + this.#keyManager = new KeyManager(rootKey) this.#deviceId = getDeviceId(this.#keyManager) this.#projectSettingsIndexWriter = new IndexWriter({ @@ -456,4 +474,36 @@ export class MapeoManager { get invite() { return this.#invite } + + /** + * @returns {Promise} + */ + async listLocalPeers() { + return omitPeerProtomux(this.#localPeers.peers) + } +} + +// We use the `protomux` property of connected peers internally, but we don't +// expose it to the API. I have avoided using a private symbol for this for fear +// that we could accidentally keep references around of protomux instances, +// which could cause a memory leak (it shouldn't, but just to eliminate the +// possibility) + +/** + * Remove the protomux property of connected peers + * + * @param {import('./local-peers.js').PeerInfo[]} peers + * @returns {PublicPeerInfo[]} + */ +function omitPeerProtomux(peers) { + return peers.map( + ({ + // @ts-ignore + // eslint-disable-next-line no-unused-vars + protomux, + ...publicPeerInfo + }) => { + return publicPeerInfo + } + ) } From 9bff9acb36a2f46aa87c740fb950f8d88cd3b49a Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 27 Oct 2023 13:35:12 +0900 Subject: [PATCH 14/69] feat: add $sync API methods For now this simplifies the API (because we are only supporting local sync, not remote sync over the internet) to: - `project.$sync.getState()` - `project.$sync.start()` - `project.$sync.stop()` - Events - `sync-state` It's currently not possible to stop local discovery, nor is it possible to stop sync of the metadata namespaces (auth, config, blobIndex). The start and stop methods stop the sync of the data and blob namespaces. Fixes #134. Stacked on #360, #358 and #356. --- src/mapeo-manager.js | 12 ++++-- src/mapeo-project.js | 35 +++++++++++----- src/sync/sync-api.js | 82 +++++++++++++++++++++++++++++++++++++ src/sync/sync-controller.js | 44 -------------------- 4 files changed, 115 insertions(+), 58 deletions(-) create mode 100644 src/sync/sync-api.js delete mode 100644 src/sync/sync-controller.js diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 999226620..2ac56b057 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -40,6 +40,7 @@ const CLIENT_SQLITE_FILE_NAME = 'client.db' const MAX_FILE_DESCRIPTORS = 768 export const kRPC = Symbol('rpc') +export const kManagerReplicate = Symbol('replicate manager') /** * @typedef {Omit} PublicPeerInfo @@ -129,7 +130,7 @@ export class MapeoManager extends TypedEmitter { this.#localDiscovery = new LocalDiscovery({ identityKeypair: this.#keyManager.getIdentityKeypair(), }) - this.#localDiscovery.on('connection', this.replicate.bind(this)) + this.#localDiscovery.on('connection', this[kManagerReplicate].bind(this)) } /** @@ -140,12 +141,15 @@ export class MapeoManager extends TypedEmitter { } /** - * Replicate Mapeo to a `@hyperswarm/secret-stream`. Should only be used for - * local (trusted) connections, because the RPC channel key is public + * Replicate Mapeo to a `@hyperswarm/secret-stream`. This replication connects + * the Mapeo RPC channel and allows invites. All active projects will sync + * automatically to this replication stream. Only use for local (trusted) + * connections, because the RPC channel key is public. To sync a specific + * project without connecting RPC, use project[kProjectReplication]. * * @param {import('@hyperswarm/secret-stream')} noiseStream */ - replicate(noiseStream) { + [kManagerReplicate](noiseStream) { const replicationStream = this.#localPeers.connect(noiseStream) Promise.all([this.getDeviceInfo(), openedNoiseSecretStream(noiseStream)]) .then(([{ name }, openedNoiseStream]) => { diff --git a/src/mapeo-project.js b/src/mapeo-project.js index cf9805174..669922966 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -32,7 +32,7 @@ import { import { Capabilities } from './capabilities.js' import { getDeviceId, projectKeyToId, valueOf } from './utils.js' import { MemberApi } from './member-api.js' -import { SyncController } from './sync/sync-controller.js' +import { SyncApi, kSyncReplicate } from './sync/sync-api.js' import Hypercore from 'hypercore' /** @typedef {Omit} EditableProjectSettings */ @@ -42,7 +42,7 @@ const INDEXER_STORAGE_FOLDER_NAME = 'indexer' export const kCoreOwnership = Symbol('coreOwnership') export const kCapabilities = Symbol('capabilities') export const kSetOwnDeviceInfo = Symbol('kSetOwnDeviceInfo') -export const kReplicate = Symbol('replicate') +export const kProjectReplicate = Symbol('replicate project') export class MapeoProject { #projectId @@ -56,7 +56,7 @@ export class MapeoProject { #capabilities #ownershipWriteDone #memberApi - #syncController + #syncApi /** * @param {Object} opts @@ -245,15 +245,17 @@ export class MapeoProject { }, }) - this.#syncController = new SyncController({ + this.#syncApi = new SyncApi({ coreManager: this.#coreManager, capabilities: this.#capabilities, }) + ///////// 4. Wire up sync + // Replicate already connected local peers for (const peer of localPeers.peers) { if (peer.status !== 'connected') continue - this.#syncController.replicate(peer.protomux) + this.#syncApi[kSyncReplicate](peer.protomux) } localPeers.on('discovery-key', (discoveryKey, stream) => { @@ -267,10 +269,10 @@ export class MapeoProject { // When a new peer is found, try to replicate (if it is not a member of the // project it will fail the capability check and be ignored) localPeers.on('peer-add', (peer) => { - this.#syncController.replicate(peer.protomux) + this.#syncApi[kSyncReplicate](peer.protomux) }) - ///////// 4. Write core ownership record + ///////// 5. Write core ownership record const deferred = pDefer() // Avoid uncaught rejection. If this is rejected then project.ready() will reject @@ -365,6 +367,10 @@ export class MapeoProject { return this.#memberApi } + get $sync() { + return this.#syncApi + } + /** * @param {Partial} settings * @returns {Promise} @@ -416,15 +422,24 @@ export class MapeoProject { } /** + * Replicate a project to a @hyperswarm/secret-stream. Invites will not + * function because the RPC channel is not connected for project replication, + * and only this project will replicate (to replicate multiple projects you + * need to replicate the manager instance via manager[kManagerReplicate]) * * @param {Exclude[0], boolean>} stream A duplex stream, a @hyperswarm/secret-stream, or a Protomux instance * @returns */ - [kReplicate](stream) { - const replicationStream = Hypercore.createProtocolStream(stream, {}) + [kProjectReplicate](stream) { + const replicationStream = Hypercore.createProtocolStream(stream, { + ondiscoverykey: async (discoveryKey) => { + this.#coreManager.handleDiscoveryKey(discoveryKey, replicationStream) + }, + }) const protomux = replicationStream.noiseStream.userData // @ts-ignore - got fed up jumping through hoops to keep TS heppy - return this.#syncController.replicate(protomux) + this.#syncApi[kSyncReplicate](protomux) + return replicationStream } /** diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js new file mode 100644 index 000000000..9508adc13 --- /dev/null +++ b/src/sync/sync-api.js @@ -0,0 +1,82 @@ +import { TypedEmitter } from 'tiny-typed-emitter' +import { SyncState } from './sync-state.js' +import { PeerSyncController } from './peer-sync-controller.js' + +export const kSyncReplicate = Symbol('replicate sync') + +/** + * @typedef {object} SyncEvents + * @property {(syncState: import('./sync-state.js').State) => void} sync-state + */ + +/** + * @extends {TypedEmitter} + */ +export class SyncApi extends TypedEmitter { + syncState + #coreManager + #capabilities + /** @type {Map} */ + #peerSyncControllers = new Map() + /** @type {Set<'local' | 'remote'>} */ + #dataSyncEnabled = new Set() + + /** + * + * @param {object} opts + * @param {import('../core-manager/index.js').CoreManager} opts.coreManager + * @param {import("../capabilities.js").Capabilities} opts.capabilities + * @param {number} [opts.throttleMs] + */ + constructor({ coreManager, throttleMs = 200, capabilities }) { + super() + this.#coreManager = coreManager + this.#capabilities = capabilities + this.syncState = new SyncState({ coreManager, throttleMs }) + this.syncState.on('state', this.emit.bind(this, 'sync-state')) + } + + getState() { + return this.syncState.getState() + } + + /** + * Start syncing data cores + */ + start() { + if (this.#dataSyncEnabled.has('local')) return + this.#dataSyncEnabled.add('local') + for (const peerSyncController of this.#peerSyncControllers.values()) { + peerSyncController.enableDataSync() + } + } + + /** + * Stop syncing data cores (metadata cores will continue syncing in the background) + */ + stop() { + if (!this.#dataSyncEnabled.has('local')) return + this.#dataSyncEnabled.delete('local') + for (const peerSyncController of this.#peerSyncControllers.values()) { + peerSyncController.disableDataSync() + } + } + + /** + * @param {import('protomux')} protomux A protomux instance + */ + [kSyncReplicate](protomux) { + if (this.#peerSyncControllers.has(protomux)) return + + const peerSyncController = new PeerSyncController({ + protomux, + coreManager: this.#coreManager, + syncState: this.syncState, + capabilities: this.#capabilities, + }) + if (this.#dataSyncEnabled.has('local')) { + peerSyncController.enableDataSync() + } + this.#peerSyncControllers.set(protomux, peerSyncController) + } +} diff --git a/src/sync/sync-controller.js b/src/sync/sync-controller.js deleted file mode 100644 index a32b6cb95..000000000 --- a/src/sync/sync-controller.js +++ /dev/null @@ -1,44 +0,0 @@ -import { TypedEmitter } from 'tiny-typed-emitter' -import { SyncState } from './sync-state.js' -import { PeerSyncController } from './peer-sync-controller.js' - -export class SyncController extends TypedEmitter { - #syncState - #coreManager - #capabilities - /** @type {Map} */ - #peerSyncControllers = new Map() - - /** - * - * @param {object} opts - * @param {import('../core-manager/index.js').CoreManager} opts.coreManager - * @param {import("../capabilities.js").Capabilities} opts.capabilities - * @param {number} [opts.throttleMs] - */ - constructor({ coreManager, throttleMs = 200, capabilities }) { - super() - this.#coreManager = coreManager - this.#capabilities = capabilities - this.#syncState = new SyncState({ coreManager, throttleMs }) - } - - getState() { - return this.#syncState.getState() - } - - /** - * @param {import('protomux')} protomux A protomux instance - */ - replicate(protomux) { - if (this.#peerSyncControllers.has(protomux)) return - - const peerSyncController = new PeerSyncController({ - protomux, - coreManager: this.#coreManager, - syncState: this.#syncState, - capabilities: this.#capabilities, - }) - this.#peerSyncControllers.set(protomux, peerSyncController) - } -} From 1c0dc6ba2edb60fafaa83358b1e7cf600b8a8af6 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 27 Oct 2023 16:54:59 +0900 Subject: [PATCH 15/69] feat: Add project.$waitForInitialSync() method Fixes Add project method to download auth + config cores #233 Rather than call this inside the `client.addProject()` method, instead I think it is better for the API consumer to call `project.$waitForInitialSync()` after adding a project, since this allows the implementer to give user feedback about what is happening. --- src/capabilities.js | 2 ++ src/mapeo-manager.js | 12 ++++--- src/mapeo-project.js | 61 +++++++++++++++++++++++++++++++- src/sync/namespace-sync-state.js | 6 +++- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/src/capabilities.js b/src/capabilities.js index 05d8b4485..666b8687e 100644 --- a/src/capabilities.js +++ b/src/capabilities.js @@ -147,6 +147,8 @@ export class Capabilities { #projectCreatorAuthCoreId #ownDeviceId + static NO_ROLE_CAPABILITIES = NO_ROLE_CAPABILITIES + /** * * @param {object} opts diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 2ac56b057..6f1cbb79a 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -383,6 +383,10 @@ export class MapeoManager extends TypedEmitter { } /** + * Add a project to this device. After adding a project the client should + * await `project.$waitForInitialSync()` to ensure that the device has + * downloaded their proof of project membership and the project config. + * * @param {import('./generated/rpc.js').Invite} invite * @returns {Promise} */ @@ -410,10 +414,9 @@ export class MapeoManager extends TypedEmitter { throw new Error(`Project with ID ${projectPublicId} already exists`) } - // TODO: Relies on completion of https://github.com/digidem/mapeo-core-next/issues/233 - // 3. Sync auth + config cores + // No awaits here - need to update table in same tick as the projectExists check - // 4. Update the project keys table + // 3. Update the project keys table this.#saveToProjectKeysTable({ projectId, projectPublicId, @@ -424,9 +427,8 @@ export class MapeoManager extends TypedEmitter { projectInfo, }) - // 5. Write device info into project + // 4. Write device info into project const deviceInfo = await this.getDeviceInfo() - if (deviceInfo.name) { const project = await this.getProject(projectPublicId) await project[kSetOwnDeviceInfo]({ name: deviceInfo.name }) diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 669922966..669d1453a 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -43,6 +43,7 @@ export const kCoreOwnership = Symbol('coreOwnership') export const kCapabilities = Symbol('capabilities') export const kSetOwnDeviceInfo = Symbol('kSetOwnDeviceInfo') export const kProjectReplicate = Symbol('replicate project') +const EMPTY_PROJECT_SETTINGS = Object.freeze({}) export class MapeoProject { #projectId @@ -413,7 +414,7 @@ export class MapeoProject { await this.#dataTypes.projectSettings.getByDocId(this.#projectId) ) } catch { - return /** @type {EditableProjectSettings} */ ({}) + return /** @type {EditableProjectSettings} */ (EMPTY_PROJECT_SETTINGS) } } @@ -421,6 +422,64 @@ export class MapeoProject { return this.#capabilities.getCapabilities(this.#deviceId) } + /** + * Sync initial data: the `auth` cores which contain the capability messages, + * and the `config` cores which contain the project name & custom config (if + * it exists). The API consumer should await this after `client.addProject()` + * to ensure that the device is fully added to the project. + * + * @param {object} [opts] + * @param {number} [opts.timeoutMs=5000] Timeout in milliseconds for max time + * to wait between sync status updates before giving up. As long as syncing is + * happening, this will never timeout, but if more than timeoutMs passes + * without any sync activity, then this will resolve `false` e.g. data has not + * synced + * @returns + */ + async $waitForInitialSync({ timeoutMs = 5000 } = {}) { + const [capability, projectSettings] = await Promise.all([ + this.$getOwnCapabilities(), + this.$getProjectSettings(), + ]) + const { + auth: { localState: authState }, + config: { localState: configState }, + } = this.#syncApi.getState() + const isCapabilitySynced = capability !== Capabilities.NO_ROLE_CAPABILITIES + const isProjectSettingsSynced = projectSettings !== EMPTY_PROJECT_SETTINGS + // Assumes every project that someone is invited to has at least one record + // in the auth store - the capability record for the invited device + const isAuthSynced = authState.want === 0 && authState.have > 0 + // Assumes every project that someone is invited to has at least one record + // in the config store - defining the name of the project. + // TODO: Enforce adding a project name in the invite method + const isConfigSynced = configState.want === 0 && configState.have > 0 + if ( + isCapabilitySynced && + isProjectSettingsSynced && + isAuthSynced && + isConfigSynced + ) { + return true + } + return new Promise((resolve) => { + /** @param {import('./sync/sync-state.js').State} syncState */ + const onSyncState = (syncState) => { + clearTimeout(timeoutId) + timeoutId = setTimeout(onTimeout, timeoutMs) + if (syncState.auth.dataToSync || syncState.config.dataToSync) return + this.#syncApi.off('sync-state', onSyncState) + resolve(this.$waitForInitialSync()) + } + const onTimeout = () => { + this.#syncApi.off('sync-state', onSyncState) + resolve(false) + } + let timeoutId = setTimeout(onTimeout, timeoutMs) + this.#syncApi.on('sync-state', onSyncState) + }) + } + /** * Replicate a project to a @hyperswarm/secret-stream. Invites will not * function because the RPC channel is not connected for project replication, diff --git a/src/sync/namespace-sync-state.js b/src/sync/namespace-sync-state.js index 3ffd6769f..c69902ca9 100644 --- a/src/sync/namespace-sync-state.js +++ b/src/sync/namespace-sync-state.js @@ -2,7 +2,7 @@ import { CoreSyncState } from './core-sync-state.js' import { discoveryKey } from 'hypercore-crypto' /** - * @typedef {Omit} SyncState + * @typedef {Omit & { dataToSync: boolean }} SyncState */ /** @@ -55,6 +55,7 @@ export class NamespaceSyncState { if (this.#cachedState) return this.#cachedState /** @type {SyncState} */ const state = { + dataToSync: false, localState: createState(), remoteStates: {}, } @@ -71,6 +72,9 @@ export class NamespaceSyncState { } } } + if (state.localState.want > 0 || state.localState.wanted > 0) { + state.dataToSync = true + } this.#cachedState = state return state } From 1f4fad25c0024ea53ab9102535153522d527dfaa Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 27 Oct 2023 17:46:22 +0900 Subject: [PATCH 16/69] Wait for initial sync within addProject() --- src/mapeo-manager.js | 91 +++++++++++++++++++++++++++++++++++++++++--- src/mapeo-project.js | 60 +---------------------------- 2 files changed, 88 insertions(+), 63 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 6f1cbb79a..a08ceaaf8 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -28,6 +28,7 @@ import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' import { LocalDiscovery } from './discovery/local-discovery.js' import { TypedEmitter } from 'tiny-typed-emitter' +import { Capabilities } from './capabilities.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -427,16 +428,96 @@ export class MapeoManager extends TypedEmitter { projectInfo, }) - // 4. Write device info into project - const deviceInfo = await this.getDeviceInfo() - if (deviceInfo.name) { + // Any errors from here we need to remove project from db because it has not + // been fully added and synced + try { + // 4. Write device info into project const project = await this.getProject(projectPublicId) - await project[kSetOwnDeviceInfo]({ name: deviceInfo.name }) - } + try { + const deviceInfo = await this.getDeviceInfo() + if (deviceInfo.name) { + await project[kSetOwnDeviceInfo]({ name: deviceInfo.name }) + } + } catch (e) { + // Can ignore an error trying to write device info + console.error(e) + } + + // 5. Wait for initial project sync + await this.#waitForInitialSync(project) + + this.#activeProjects.set(projectPublicId, project) + } catch (e) { + this.#db + .delete(projectKeysTable) + .where(eq(projectKeysTable.projectId, projectId)) + .run() + throw e + } return projectPublicId } + /** + * Sync initial data: the `auth` cores which contain the capability messages, + * and the `config` cores which contain the project name & custom config (if + * it exists). The API consumer should await this after `client.addProject()` + * to ensure that the device is fully added to the project. + * + * @param {MapeoProject} project + * @param {object} [opts] + * @param {number} [opts.timeoutMs=5000] Timeout in milliseconds for max time + * to wait between sync status updates before giving up. As long as syncing is + * happening, this will never timeout, but if more than timeoutMs passes + * without any sync activity, then this will resolve `false` e.g. data has not + * synced + * @returns + */ + async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) { + const [capability, projectSettings] = await Promise.all([ + project.$getOwnCapabilities(), + project.$getProjectSettings(), + ]) + const { + auth: { localState: authState }, + config: { localState: configState }, + } = project.$sync.getState() + const isCapabilitySynced = capability !== Capabilities.NO_ROLE_CAPABILITIES + const isProjectSettingsSynced = + projectSettings !== MapeoProject.EMPTY_PROJECT_SETTINGS + // Assumes every project that someone is invited to has at least one record + // in the auth store - the capability record for the invited device + const isAuthSynced = authState.want === 0 && authState.have > 0 + // Assumes every project that someone is invited to has at least one record + // in the config store - defining the name of the project. + // TODO: Enforce adding a project name in the invite method + const isConfigSynced = configState.want === 0 && configState.have > 0 + if ( + isCapabilitySynced && + isProjectSettingsSynced && + isAuthSynced && + isConfigSynced + ) { + return true + } + return new Promise((resolve, reject) => { + /** @param {import('./sync/sync-state.js').State} syncState */ + const onSyncState = (syncState) => { + clearTimeout(timeoutId) + timeoutId = setTimeout(onTimeout, timeoutMs) + if (syncState.auth.dataToSync || syncState.config.dataToSync) return + project.$sync.off('sync-state', onSyncState) + resolve(this.#waitForInitialSync(project, { timeoutMs })) + } + const onTimeout = () => { + project.$sync.off('sync-state', onSyncState) + reject(new Error('Sync timeout')) + } + let timeoutId = setTimeout(onTimeout, timeoutMs) + project.$sync.on('sync-state', onSyncState) + }) + } + /** * @template {import('type-fest').Exact} T * @param {T} deviceInfo diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 669d1453a..281758d1d 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -59,6 +59,8 @@ export class MapeoProject { #memberApi #syncApi + static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS + /** * @param {Object} opts * @param {string} opts.dbPath Path to store project sqlite db. Use `:memory:` for memory storage @@ -422,64 +424,6 @@ export class MapeoProject { return this.#capabilities.getCapabilities(this.#deviceId) } - /** - * Sync initial data: the `auth` cores which contain the capability messages, - * and the `config` cores which contain the project name & custom config (if - * it exists). The API consumer should await this after `client.addProject()` - * to ensure that the device is fully added to the project. - * - * @param {object} [opts] - * @param {number} [opts.timeoutMs=5000] Timeout in milliseconds for max time - * to wait between sync status updates before giving up. As long as syncing is - * happening, this will never timeout, but if more than timeoutMs passes - * without any sync activity, then this will resolve `false` e.g. data has not - * synced - * @returns - */ - async $waitForInitialSync({ timeoutMs = 5000 } = {}) { - const [capability, projectSettings] = await Promise.all([ - this.$getOwnCapabilities(), - this.$getProjectSettings(), - ]) - const { - auth: { localState: authState }, - config: { localState: configState }, - } = this.#syncApi.getState() - const isCapabilitySynced = capability !== Capabilities.NO_ROLE_CAPABILITIES - const isProjectSettingsSynced = projectSettings !== EMPTY_PROJECT_SETTINGS - // Assumes every project that someone is invited to has at least one record - // in the auth store - the capability record for the invited device - const isAuthSynced = authState.want === 0 && authState.have > 0 - // Assumes every project that someone is invited to has at least one record - // in the config store - defining the name of the project. - // TODO: Enforce adding a project name in the invite method - const isConfigSynced = configState.want === 0 && configState.have > 0 - if ( - isCapabilitySynced && - isProjectSettingsSynced && - isAuthSynced && - isConfigSynced - ) { - return true - } - return new Promise((resolve) => { - /** @param {import('./sync/sync-state.js').State} syncState */ - const onSyncState = (syncState) => { - clearTimeout(timeoutId) - timeoutId = setTimeout(onTimeout, timeoutMs) - if (syncState.auth.dataToSync || syncState.config.dataToSync) return - this.#syncApi.off('sync-state', onSyncState) - resolve(this.$waitForInitialSync()) - } - const onTimeout = () => { - this.#syncApi.off('sync-state', onSyncState) - resolve(false) - } - let timeoutId = setTimeout(onTimeout, timeoutMs) - this.#syncApi.on('sync-state', onSyncState) - }) - } - /** * Replicate a project to a @hyperswarm/secret-stream. Invites will not * function because the RPC channel is not connected for project replication, From d00bf109e733faf1be425087772ef32d59fa3b43 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 28 Oct 2023 09:51:15 +0900 Subject: [PATCH 17/69] fix: don't add core bitfield until core is ready --- src/sync/core-sync-state.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/sync/core-sync-state.js b/src/sync/core-sync-state.js index ec382656f..6872dd53c 100644 --- a/src/sync/core-sync-state.js +++ b/src/sync/core-sync-state.js @@ -93,10 +93,13 @@ export class CoreSyncState { if (this.#core) return this.#core = core - this.#localState.setHavesBitfield( - // @ts-ignore - internal property - core?.core?.bitfield - ) + + this.#core.ready().then(() => { + this.#localState.setHavesBitfield( + // @ts-ignore - internal property + core?.core?.bitfield + ) + }) for (const peer of this.#core.peers) { this.#onPeerAdd(peer) From 989b4299ec49617eba2852ed6f6ebd7932a8f595 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 28 Oct 2023 09:52:24 +0900 Subject: [PATCH 18/69] feat: expose deviceId on coreManager --- src/core-manager/index.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index e548f2d94..9ee0df166 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -54,6 +54,7 @@ export class CoreManager extends TypedEmitter { #state = 'opened' #ready #haveExtension + #deviceId static get namespaces() { return NAMESPACES @@ -86,6 +87,7 @@ export class CoreManager extends TypedEmitter { 'project owner core secret key must be 64-byte buffer' ) const primaryKey = keyManager.getDerivedKey('primaryKey', projectKey) + this.#deviceId = keyManager.getIdentityKeypair().publicKey.toString('hex') this.#projectKey = projectKey this.#encryptionKeys = encryptionKeys @@ -160,6 +162,10 @@ export class CoreManager extends TypedEmitter { ).catch(() => {}) } + get deviceId() { + return this.#deviceId + } + get creatorCore() { return this.#creatorCore } From bbda3f0769b5233928199f266deb61078aab6d64 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 28 Oct 2023 09:53:24 +0900 Subject: [PATCH 19/69] fix: wait for project.ready() in waitForInitialSync --- src/mapeo-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index a08ceaaf8..eef3d5d45 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -474,6 +474,7 @@ export class MapeoManager extends TypedEmitter { * @returns */ async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) { + await project.ready() const [capability, projectSettings] = await Promise.all([ project.$getOwnCapabilities(), project.$getProjectSettings(), From 0a32989674fd1195dd06cad1a6d5f4a0630d788d Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 28 Oct 2023 16:54:37 +0900 Subject: [PATCH 20/69] fix: skip waitForSync in tests --- src/mapeo-manager.js | 43 +++++++++++++++++++------- test-e2e/capabilities.js | 22 +++++++++----- test-e2e/device-info.js | 11 ++++--- test-e2e/manager-basic.js | 63 ++++++++++++++++++++++++--------------- test-e2e/members.js | 11 ++++--- 5 files changed, 99 insertions(+), 51 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index eef3d5d45..b6d583cb3 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -29,6 +29,7 @@ import { InviteApi } from './invite-api.js' import { LocalDiscovery } from './discovery/local-discovery.js' import { TypedEmitter } from 'tiny-typed-emitter' import { Capabilities } from './capabilities.js' +import NoiseSecretStream from '@hyperswarm/secret-stream' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -131,7 +132,7 @@ export class MapeoManager extends TypedEmitter { this.#localDiscovery = new LocalDiscovery({ identityKeypair: this.#keyManager.getIdentityKeypair(), }) - this.#localDiscovery.on('connection', this[kManagerReplicate].bind(this)) + this.#localDiscovery.on('connection', this.#replicate.bind(this)) } /** @@ -141,16 +142,30 @@ export class MapeoManager extends TypedEmitter { return this.#localPeers } + get deviceId() { + return this.#deviceId + } + /** - * Replicate Mapeo to a `@hyperswarm/secret-stream`. This replication connects - * the Mapeo RPC channel and allows invites. All active projects will sync - * automatically to this replication stream. Only use for local (trusted) - * connections, because the RPC channel key is public. To sync a specific - * project without connecting RPC, use project[kProjectReplication]. + * Create a Mapeo replication stream. This replication connects the Mapeo RPC + * channel and allows invites. All active projects will sync automatically to + * this replication stream. Only use for local (trusted) connections, because + * the RPC channel key is public. To sync a specific project without + * connecting RPC, use project[kProjectReplication]. * - * @param {import('@hyperswarm/secret-stream')} noiseStream + * @param {boolean} isInitiator + */ + [kManagerReplicate](isInitiator) { + const noiseStream = new NoiseSecretStream(isInitiator, undefined, { + keyPair: this.#keyManager.getIdentityKeypair(), + }) + return this.#replicate(noiseStream) + } + + /** + * @param {NoiseSecretStream} noiseStream */ - [kManagerReplicate](noiseStream) { + #replicate(noiseStream) { const replicationStream = this.#localPeers.connect(noiseStream) Promise.all([this.getDeviceInfo(), openedNoiseSecretStream(noiseStream)]) .then(([{ name }, openedNoiseStream]) => { @@ -389,9 +404,13 @@ export class MapeoManager extends TypedEmitter { * downloaded their proof of project membership and the project config. * * @param {import('./generated/rpc.js').Invite} invite + * @param {{ waitForSync?: boolean }} [opts] For internal use in tests, set opts.waitForSync = false to not wait for sync during addProject() * @returns {Promise} */ - async addProject({ projectKey, encryptionKeys, projectInfo }) { + async addProject( + { projectKey, encryptionKeys, projectInfo }, + { waitForSync = true } = {} + ) { const projectPublicId = projectKeyToPublicId(projectKey) // 1. Check for an active project @@ -445,7 +464,9 @@ export class MapeoManager extends TypedEmitter { } // 5. Wait for initial project sync - await this.#waitForInitialSync(project) + if (waitForSync) { + await this.#waitForInitialSync(project) + } this.#activeProjects.set(projectPublicId, project) } catch (e) { @@ -495,7 +516,7 @@ export class MapeoManager extends TypedEmitter { const isConfigSynced = configState.want === 0 && configState.have > 0 if ( isCapabilitySynced && - isProjectSettingsSynced && + // isProjectSettingsSynced && isAuthSynced && isConfigSynced ) { diff --git a/test-e2e/capabilities.js b/test-e2e/capabilities.js index 7019857fa..2a8328979 100644 --- a/test-e2e/capabilities.js +++ b/test-e2e/capabilities.js @@ -49,10 +49,13 @@ test('New device without capabilities', async (t) => { coreStorage: () => new RAM(), }) - const projectId = await manager.addProject({ - projectKey: randomBytes(32), - encryptionKeys: { auth: randomBytes(32) }, - }) + const projectId = await manager.addProject( + { + projectKey: randomBytes(32), + encryptionKeys: { auth: randomBytes(32) }, + }, + { waitForSync: false } + ) const project = await manager.getProject(projectId) await project.ready() @@ -123,10 +126,13 @@ test('getMany() - on newly invited device before sync', async (t) => { coreStorage: () => new RAM(), }) - const projectId = await manager.addProject({ - projectKey: randomBytes(32), - encryptionKeys: { auth: randomBytes(32) }, - }) + const projectId = await manager.addProject( + { + projectKey: randomBytes(32), + encryptionKeys: { auth: randomBytes(32) }, + }, + { waitForSync: false } + ) const project = await manager.getProject(projectId) await project.ready() diff --git a/test-e2e/device-info.js b/test-e2e/device-info.js index 3a0a7844a..bc037a1a3 100644 --- a/test-e2e/device-info.js +++ b/test-e2e/device-info.js @@ -52,10 +52,13 @@ test('device info written to projects', async (t) => { await manager.setDeviceInfo({ name: 'mapeo' }) - const projectId = await manager.addProject({ - projectKey: randomBytes(32), - encryptionKeys: { auth: randomBytes(32) }, - }) + const projectId = await manager.addProject( + { + projectKey: randomBytes(32), + encryptionKeys: { auth: randomBytes(32) }, + }, + { waitForSync: false } + ) const project = await manager.getProject(projectId) diff --git a/test-e2e/manager-basic.js b/test-e2e/manager-basic.js index ac8136fb4..0a62e1010 100644 --- a/test-e2e/manager-basic.js +++ b/test-e2e/manager-basic.js @@ -113,17 +113,23 @@ test('Managing added projects', async (t) => { coreStorage: () => new RAM(), }) - const project1Id = await manager.addProject({ - projectKey: KeyManager.generateProjectKeypair().publicKey, - encryptionKeys: { auth: randomBytes(32) }, - projectInfo: { name: 'project 1' }, - }) + const project1Id = await manager.addProject( + { + projectKey: KeyManager.generateProjectKeypair().publicKey, + encryptionKeys: { auth: randomBytes(32) }, + projectInfo: { name: 'project 1' }, + }, + { waitForSync: false } + ) - const project2Id = await manager.addProject({ - projectKey: KeyManager.generateProjectKeypair().publicKey, - encryptionKeys: { auth: randomBytes(32) }, - projectInfo: { name: 'project 2' }, - }) + const project2Id = await manager.addProject( + { + projectKey: KeyManager.generateProjectKeypair().publicKey, + encryptionKeys: { auth: randomBytes(32) }, + projectInfo: { name: 'project 2' }, + }, + { waitForSync: false } + ) t.test('initial information from listed projects', async (t) => { const listedProjects = await manager.listProjects() @@ -183,11 +189,14 @@ test('Managing both created and added projects', async (t) => { name: 'created project', }) - const addedProjectId = await manager.addProject({ - projectKey: KeyManager.generateProjectKeypair().publicKey, - encryptionKeys: { auth: randomBytes(32) }, - projectInfo: { name: 'added project' }, - }) + const addedProjectId = await manager.addProject( + { + projectKey: KeyManager.generateProjectKeypair().publicKey, + encryptionKeys: { auth: randomBytes(32) }, + projectInfo: { name: 'added project' }, + }, + { waitForSync: false } + ) const listedProjects = await manager.listProjects() @@ -222,10 +231,13 @@ test('Manager cannot add project that already exists', async (t) => { const existingProjectsCountBefore = (await manager.listProjects()).length t.exception( - manager.addProject({ - projectKey: Buffer.from(existingProjectId, 'hex'), - encryptionKeys: { auth: randomBytes(32) }, - }), + manager.addProject( + { + projectKey: Buffer.from(existingProjectId, 'hex'), + encryptionKeys: { auth: randomBytes(32) }, + }, + { waitForSync: false } + ), 'attempting to add project that already exists throws' ) @@ -247,11 +259,14 @@ test('Consistent storage folders', async (t) => { }) for (let i = 0; i < 10; i++) { - const projectId = await manager.addProject({ - projectKey: randomBytesSeed('test' + i), - encryptionKeys: { auth: randomBytes(32) }, - projectInfo: {}, - }) + const projectId = await manager.addProject( + { + projectKey: randomBytesSeed('test' + i), + encryptionKeys: { auth: randomBytes(32) }, + projectInfo: {}, + }, + { waitForSync: false } + ) await manager.getProject(projectId) } diff --git a/test-e2e/members.js b/test-e2e/members.js index 64c28dee9..5a951c852 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -51,10 +51,13 @@ test('getting yourself after being invited to project (but not yet synced)', asy await manager.setDeviceInfo({ name: 'mapeo' }) const project = await manager.getProject( - await manager.addProject({ - projectKey: randomBytes(32), - encryptionKeys: { auth: randomBytes(32) }, - }) + await manager.addProject( + { + projectKey: randomBytes(32), + encryptionKeys: { auth: randomBytes(32) }, + }, + { waitForSync: false } + ) ) await project.ready() From dc7a7c38784bec4c07406d366bfb6cc6e7626942 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 11:32:45 +0900 Subject: [PATCH 21/69] don't enable/disable namespace if not needed --- src/sync/peer-sync-controller.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index 01cf76ff7..8a5621ed6 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -217,6 +217,7 @@ export class PeerSyncController { * @param {Namespace} namespace */ #enableNamespace(namespace) { + if (this.#enabledNamespaces.has(namespace)) return for (const { core } of this.#coreManager.getCores(namespace)) { this.#replicateCore(core) this.#downloadCore(core) @@ -228,6 +229,7 @@ export class PeerSyncController { * @param {Namespace} namespace */ #disableNamespace(namespace) { + if (!this.#enabledNamespaces.has(namespace)) return for (const { core } of this.#coreManager.getCores(namespace)) { this.#unreplicateCore(core) this.#undownloadCore(core) From 1895679ef93cc158f12870c5f22e1b37ca6d092e Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 11:35:55 +0900 Subject: [PATCH 22/69] start core download when created via sparse: false --- src/core-manager/index.js | 2 ++ src/sync/peer-sync-controller.js | 21 --------------------- types/corestore.d.ts | 1 + 3 files changed, 3 insertions(+), 21 deletions(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index 9ee0df166..cbe1d29a8 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -269,6 +269,8 @@ export class CoreManager extends TypedEmitter { const core = this.#corestore.get({ keyPair, encryptionKey: this.#encryptionKeys[namespace], + // Starts live download of core immediately + sparse: false, }) // @ts-ignore - ensure key is defined before hypercore is ready core.key = key diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index 8a5621ed6..f91a0e204 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -194,25 +194,6 @@ export class PeerSyncController { this.#replicatingCores.delete(core) } - /** - * @param {import('hypercore')<'binary', any>} core - */ - #downloadCore(core) { - if (this.#downloadingRanges.has(core)) return - const range = core.download({ start: 0, end: -1 }) - this.#downloadingRanges.set(core, range) - } - - /** - * @param {import('hypercore')<'binary', any>} core - */ - #undownloadCore(core) { - const range = this.#downloadingRanges.get(core) - if (!range) return - range.destroy() - this.#downloadingRanges.delete(core) - } - /** * @param {Namespace} namespace */ @@ -220,7 +201,6 @@ export class PeerSyncController { if (this.#enabledNamespaces.has(namespace)) return for (const { core } of this.#coreManager.getCores(namespace)) { this.#replicateCore(core) - this.#downloadCore(core) } this.#enabledNamespaces.add(namespace) } @@ -232,7 +212,6 @@ export class PeerSyncController { if (!this.#enabledNamespaces.has(namespace)) return for (const { core } of this.#coreManager.getCores(namespace)) { this.#unreplicateCore(core) - this.#undownloadCore(core) } this.#enabledNamespaces.delete(namespace) } diff --git a/types/corestore.d.ts b/types/corestore.d.ts index a4bd6bfc2..d0518074d 100644 --- a/types/corestore.d.ts +++ b/types/corestore.d.ts @@ -27,6 +27,7 @@ declare module 'corestore' { options: Omit & { key?: Buffer | string | undefined keyPair: { publicKey: Buffer; secretKey?: Buffer | undefined | null } + sparse: boolean } ): Hypercore replicate: typeof Hypercore.prototype.replicate From 763b57b96479cb69808b07c130bd225a1f72e4dd Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 15:11:07 +0900 Subject: [PATCH 23/69] Add debug logging This was a big lift, but necessary to be able to debug sync issues since temporarily adding console.log statements was too much work, and debugging requires knowing the deviceId associated with each message. --- package-lock.json | 3 +- src/core-manager/index.js | 30 ++++++++++++-- src/index-writer/index.js | 17 +++++++- src/local-peers.js | 71 +++++++++++++++++++++++++++++--- src/logger.js | 69 +++++++++++++++++++++++++++++++ src/mapeo-manager.js | 49 +++++++++++++++++++--- src/mapeo-project.js | 12 +++++- src/sync/peer-sync-controller.js | 51 +++++++++++++++++------ src/sync/sync-api.js | 17 +++++++- 9 files changed, 286 insertions(+), 33 deletions(-) create mode 100644 src/logger.js diff --git a/package-lock.json b/package-lock.json index 319dfb60e..550b10705 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2236,7 +2236,8 @@ }, "node_modules/debug": { "version": "4.3.4", - "license": "MIT", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, diff --git a/src/core-manager/index.js b/src/core-manager/index.js index cbe1d29a8..733bb6d2a 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -8,6 +8,7 @@ import { HaveExtension, ProjectExtension } from '../generated/extensions.js' import { CoreIndex } from './core-index.js' import { ReplicationStateMachine } from './replication-state-machine.js' import * as rle from './bitfield-rle.js' +import { Logger } from '../logger.js' // WARNING: Changing these will break things for existing apps, since namespaces // are used for key derivation @@ -55,6 +56,7 @@ export class CoreManager extends TypedEmitter { #ready #haveExtension #deviceId + #l static get namespaces() { return NAMESPACES @@ -68,6 +70,7 @@ export class CoreManager extends TypedEmitter { * @param {Buffer} [options.projectSecretKey] 32-byte secret key of the project creator core * @param {Partial>} [options.encryptionKeys] Encryption keys for each namespace * @param {import('hypercore').HypercoreStorage} options.storage Folder to store all hypercore data + * @param {Logger} [options.logger] */ constructor({ sqlite, @@ -76,6 +79,7 @@ export class CoreManager extends TypedEmitter { projectSecretKey, encryptionKeys = {}, storage, + logger, }) { super() assert( @@ -86,6 +90,7 @@ export class CoreManager extends TypedEmitter { !projectSecretKey || projectSecretKey.length === 64, 'project owner core secret key must be 64-byte buffer' ) + this.#l = Logger.create('coreManager', logger) const primaryKey = keyManager.getDerivedKey('primaryKey', projectKey) this.#deviceId = keyManager.getIdentityKeypair().publicKey.toString('hex') this.#projectKey = projectKey @@ -159,7 +164,11 @@ export class CoreManager extends TypedEmitter { this.#ready = Promise.all( [...this.#coreIndex].map(({ core }) => core.ready()) - ).catch(() => {}) + ) + .then(() => { + this.#l.log('ready') + }) + .catch(() => {}) } get deviceId() { @@ -311,7 +320,12 @@ export class CoreManager extends TypedEmitter { if (persist) { this.#addCoreSqlStmt.run({ publicKey: key, namespace }) } - + this.#l.log( + 'Added %s %s core %k', + persist ? 'remote' : writer ? 'local' : 'creator', + namespace, + key + ) this.emit('add-core', { core, key, namespace }) return { core, key, namespace } @@ -407,13 +421,21 @@ export class CoreManager extends TypedEmitter { const discoveryId = discoveryKey.toString('hex') const peer = await this.#findPeer(stream.remotePublicKey) if (!peer) { - console.warn('handleDiscovery no peer', stream.remotePublicKey) + this.#l.log( + 'Receive dk %h but no connected peer %h', + discoveryKey, + stream.remotePublicKey + ) // TODO: How to handle this and when does it happen? return } // If we already know about this core, then we will add it to the // replication stream when we are ready - if (this.#coreIndex.getByDiscoveryId(discoveryId)) return + if (this.#coreIndex.getByDiscoveryId(discoveryId)) { + this.#l.log('Received dk %h, but already have core', discoveryKey) + return + } + this.#l.log('Requesting core key for dk %h', discoveryKey) const message = ProjectExtension.fromPartial({ wantCoreKeys: [discoveryKey], }) diff --git a/src/index-writer/index.js b/src/index-writer/index.js index b131e2468..a48c5fde1 100644 --- a/src/index-writer/index.js +++ b/src/index-writer/index.js @@ -3,6 +3,7 @@ import SqliteIndexer from '@mapeo/sqlite-indexer' import { getTableConfig } from 'drizzle-orm/sqlite-core' import { getBacklinkTableName } from '../schema/utils.js' import { discoveryKey } from 'hypercore-crypto' +import { Logger } from '../logger.js' /** * @typedef {import('../datatype/index.js').MapeoDocTables} MapeoDocTables @@ -21,6 +22,7 @@ export class IndexWriter { /** @type {Map} */ #indexers = new Map() #mapDoc + #l /** * * @param {object} opts @@ -28,8 +30,10 @@ export class IndexWriter { * @param {TTables[]} opts.tables * @param {(doc: MapeoDocInternal, version: import('@mapeo/schema').VersionIdObject) => MapeoDoc} [opts.mapDoc] optionally transform a document prior to indexing. Can also validate, if an error is thrown then the document will not be indexed * @param {typeof import('@mapeo/sqlite-indexer').defaultGetWinner} [opts.getWinner] custom function to determine the "winner" of two forked documents. Defaults to choosing the document with the most recent `updatedAt` + * @param {Logger} [opts.logger] */ - constructor({ tables, sqlite, mapDoc = (d) => d, getWinner }) { + constructor({ tables, sqlite, mapDoc = (d) => d, getWinner, logger }) { + this.#l = Logger.create('indexWriter', logger) this.#mapDoc = mapDoc for (const table of tables) { const config = getTableConfig(table) @@ -63,6 +67,7 @@ export class IndexWriter { const version = { coreDiscoveryKey: discoveryKey(key), index } var doc = this.#mapDoc(decode(block, version), version) } catch (e) { + this.#l.log('Could not decode entry %d of %h', index, key) // Unknown or invalid entry - silently ignore continue } @@ -80,6 +85,16 @@ export class IndexWriter { continue } indexer.batch(docs) + if (this.#l.enabled) { + for (const doc of docs) { + this.#l.log( + 'Indexed %s %S @ %S', + doc.schemaName, + doc.docId, + doc.versionId + ) + } + } } } } diff --git a/src/local-peers.js b/src/local-peers.js index 32c0d2305..aa8df597e 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -10,6 +10,7 @@ import { InviteResponse_Decision, } from './generated/rpc.js' import pDefer from 'p-defer' +import { Logger } from './logger.js' const PROTOCOL_NAME = 'mapeo/rpc' @@ -59,16 +60,23 @@ class Peer { #disconnectedAt = 0 /** @type {Protomux} */ #protomux + #log /** * @param {object} options * @param {Buffer} options.publicKey * @param {ReturnType} options.channel + * @param {Logger} [options.logger] */ - constructor({ publicKey, channel }) { + constructor({ publicKey, channel, logger }) { this.#publicKey = publicKey this.#channel = channel this.#connected = pDefer() + // @ts-ignore + this.#log = (formatter, ...args) => { + const log = Logger.create('peer', logger).log + return log.apply(null, [`[%h] ${formatter}`, publicKey, ...args]) + } } /** @returns {PeerInfoInternal} */ get info() { @@ -108,26 +116,34 @@ class Peer { this.#protomux = protomux /* c8 ignore next 3 */ if (this.#state !== 'connecting') { + this.#log('ERROR: tried to connect but state was %s', this.#state) return // TODO: report error - this should not happen } this.#state = 'connected' this.#connectedAt = Date.now() this.#connected.resolve() + this.#log('connected') } disconnect() { // @ts-ignore - easier to ignore this than handle this for TS - avoids holding a reference to old Protomux instances this.#protomux = undefined /* c8 ignore next */ - if (this.#state === 'disconnected') return + if (this.#state === 'disconnected') { + this.#log('ERROR: tried to disconnect but was already disconnected') + return + } this.#state = 'disconnected' this.#disconnectedAt = Date.now() // Can just resolve this rather than reject, because #assertConnected will throw the error this.#connected.resolve() + let rejectCount = 0 for (const pending of this.pendingInvites.values()) { for (const { reject } of pending) { reject(new PeerDisconnectedError()) + rejectCount++ } } + this.#log('disconnected and rejected %d pending invites', rejectCount) this.pendingInvites.clear() } /** @param {InviteWithKeys} invite */ @@ -136,6 +152,7 @@ class Peer { const buf = Buffer.from(Invite.encode(invite).finish()) const messageType = MESSAGE_TYPES.Invite this.#channel.messages[messageType].send(buf) + this.#log('sent invite for %h', invite.projectKey) } /** @param {InviteResponse} response */ async sendInviteResponse(response) { @@ -143,6 +160,11 @@ class Peer { const buf = Buffer.from(InviteResponse.encode(response).finish()) const messageType = MESSAGE_TYPES.InviteResponse this.#channel.messages[messageType].send(buf) + this.#log( + 'sent response for %h: %s', + response.projectKey, + response.decision + ) } /** @param {DeviceInfo} deviceInfo */ async sendDeviceInfo(deviceInfo) { @@ -150,10 +172,12 @@ class Peer { const buf = Buffer.from(DeviceInfo.encode(deviceInfo).finish()) const messageType = MESSAGE_TYPES.DeviceInfo this.#channel.messages[messageType].send(buf) + this.#log('sent deviceInfo %o', deviceInfo) } /** @param {DeviceInfo} deviceInfo */ receiveDeviceInfo(deviceInfo) { this.#name = deviceInfo.name + this.#log('received deviceInfo %o', deviceInfo) } async #assertConnected() { await this.#connected.promise @@ -179,6 +203,17 @@ export class LocalPeers extends TypedEmitter { #opening = new Set() static InviteResponse = InviteResponse_Decision + #l + + /** + * + * @param {object} opts + * @param {Logger} [opts.logger] + */ + constructor({ logger }) { + super() + this.#l = Logger.create('localPeers', logger) + } /** * Invite a peer to a project. Resolves with the response from the invitee: @@ -195,7 +230,6 @@ export class LocalPeers extends TypedEmitter { async invite(peerId, { timeout, ...invite }) { await Promise.all(this.#opening) const peer = this.#peers.get(peerId) - if (!peer) console.log([...this.#peers.keys()]) if (!peer) throw new UnknownPeerError('Unknown peer ' + peerId) /** @type {Promise} */ return new Promise((origResolve, origReject) => { @@ -277,6 +311,11 @@ export class LocalPeers extends TypedEmitter { protomux.pair( { protocol: 'hypercore/alpha' }, /** @param {Buffer} discoveryKey */ async (discoveryKey) => { + this.#l.log( + 'Received dk %h from %h', + discoveryKey, + stream.noiseStream.remotePublicKey + ) this.emit('discovery-key', discoveryKey, stream.rawStream) } ) @@ -288,7 +327,13 @@ export class LocalPeers extends TypedEmitter { // opened, so this helped awaits the open openedNoiseSecretStream(stream).then((stream) => { this.#opening.delete(stream.opened) - if (stream.destroyed) return + if (stream.destroyed) { + this.#l.log( + 'Opened connection to %h but was already destroyed', + stream.remotePublicKey + ) + return + } const { remotePublicKey } = stream // This is written like this because the protomux uses the index within @@ -318,7 +363,11 @@ export class LocalPeers extends TypedEmitter { if (existingPeer && existingPeer.info.status !== 'disconnected') { existingPeer.disconnect() // Should not happen, but in case } - const peer = new Peer({ publicKey: remotePublicKey, channel }) + const peer = new Peer({ + publicKey: remotePublicKey, + channel, + logger: this.#l, + }) this.#peers.set(peerId, peer) // Do not emit peers now - will emit when connected }) @@ -345,7 +394,10 @@ export class LocalPeers extends TypedEmitter { const peerId = publicKey.toString('hex') const peer = this.#peers.get(peerId) /* c8 ignore next */ - if (!peer) return // TODO: report error - this should not happen + if (!peer) { + this.#l.log('ERROR: Could not close peer %h', publicKey) + return // TODO: report error - this should not happen + } // No-op if no change in state /* c8 ignore next */ if (peer.info.status === 'disconnected') return @@ -384,6 +436,7 @@ export class LocalPeers extends TypedEmitter { const invite = Invite.decode(value) assertInviteHasKeys(invite) this.emit('invite', peerId, invite) + this.#l.log('Invite from %h for %h', peerPublicKey, invite.projectKey) break } case 'InviteResponse': { @@ -397,6 +450,12 @@ export class LocalPeers extends TypedEmitter { for (const deferredPromise of pending) { deferredPromise.resolve(response.decision) } + this.#l.log( + 'Invite response from %h for %h: %s', + peerPublicKey, + response.projectKey, + response.decision + ) peer.pendingInvites.set(projectId, []) break } diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 000000000..356e6cc35 --- /dev/null +++ b/src/logger.js @@ -0,0 +1,69 @@ +import createDebug from 'debug' +import { discoveryKey } from 'hypercore-crypto' + +const TRIM = 7 + +createDebug.formatters.h = (v) => { + if (!Buffer.isBuffer(v)) return '[undefined]' + return v.toString('hex').slice(0, TRIM) +} + +createDebug.formatters.S = (v) => { + if (typeof v !== 'string') return '[undefined]' + return v.slice(0, 7) +} + +createDebug.formatters.k = (v) => { + if (!Buffer.isBuffer(v)) return '[undefined]' + return discoveryKey(v).toString('hex').slice(0, TRIM) +} + +const counts = new Map() + +export class Logger { + #baseLogger + #log + + /** + * @param {string} ns + * @param {Logger} [logger] + */ + static create(ns, logger) { + if (logger) return logger.extend(ns) + const i = (counts.get(ns) || 0) + 1 + const deviceId = String(i).padStart(TRIM, '0') + return new Logger({ deviceId, ns }) + } + + /** + * @param {object} opts + * @param {string} opts.deviceId + * @param {createDebug.Debugger} [opts.baseLogger] + * @param {string} [opts.ns] + */ + constructor({ deviceId, baseLogger, ns }) { + this.deviceId = deviceId + this.#baseLogger = baseLogger || createDebug('mapeo' + (ns ? `:${ns}` : '')) + this.#log = this.#baseLogger.extend(this.deviceId.slice(0, TRIM)) + } + get enabled() { + return this.#log.enabled + } + + /** + * @param {Parameters} args + */ + log = (...args) => { + this.#log.apply(this, args) + } + /** + * + * @param {string} ns + */ + extend(ns) { + return new Logger({ + deviceId: this.deviceId, + baseLogger: this.#baseLogger.extend(ns), + }) + } +} diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index b6d583cb3..1e65733d0 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -30,6 +30,7 @@ import { LocalDiscovery } from './discovery/local-discovery.js' import { TypedEmitter } from 'tiny-typed-emitter' import { Capabilities } from './capabilities.js' import NoiseSecretStream from '@hyperswarm/secret-stream' +import { Logger } from './logger.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -70,6 +71,7 @@ export class MapeoManager extends TypedEmitter { #localPeers #invite #localDiscovery + #l /** * @param {Object} opts @@ -79,6 +81,9 @@ export class MapeoManager extends TypedEmitter { */ constructor({ rootKey, dbFolder, coreStorage }) { super() + this.#keyManager = new KeyManager(rootKey) + this.#deviceId = getDeviceId(this.#keyManager) + this.#l = new Logger({ deviceId: this.#deviceId }) this.#dbFolder = dbFolder const sqlite = new Database( dbFolder === ':memory:' @@ -90,16 +95,20 @@ export class MapeoManager extends TypedEmitter { migrationsFolder: new URL('../drizzle/client', import.meta.url).pathname, }) - this.#localPeers = new LocalPeers() + this.#localPeers = new LocalPeers({ logger: this.#l }) this.#localPeers.on('peers', (peers) => { this.emit('local-peers', omitPeerProtomux(peers)) }) + this.#localPeers.on('discovery-key', (dk) => { + if (this.#activeProjects.size === 0) { + this.#l.log('Received dk %h but no active projects', dk) + } + }) - this.#keyManager = new KeyManager(rootKey) - this.#deviceId = getDeviceId(this.#keyManager) this.#projectSettingsIndexWriter = new IndexWriter({ tables: [projectSettingsTable], sqlite, + logger: this.#l, }) this.#activeProjects = new Map() @@ -175,7 +184,11 @@ export class MapeoManager extends TypedEmitter { }) .catch((e) => { // Ignore error but log - console.error('Failed to send device info to peer', e) + this.#l.log( + 'Failed to send device info to peer %h', + noiseStream.remotePublicKey, + e + ) }) return replicationStream } @@ -291,6 +304,12 @@ export class MapeoManager extends TypedEmitter { // TODO: Close the project instance instead of keeping it around this.#activeProjects.set(projectPublicId, project) + this.#l.log( + 'created project %h, public id: %S', + projectKeypair.publicKey, + projectPublicId + ) + // 7. Return project public id return projectPublicId } @@ -344,6 +363,7 @@ export class MapeoManager extends TypedEmitter { sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, localPeers: this.#localPeers, + logger: this.#l, }) } @@ -460,7 +480,11 @@ export class MapeoManager extends TypedEmitter { } } catch (e) { // Can ignore an error trying to write device info - console.error(e) + this.#l.log( + 'ERROR: failed to write project %h deviceInfo %o', + projectKey, + e + ) } // 5. Wait for initial project sync @@ -470,12 +494,14 @@ export class MapeoManager extends TypedEmitter { this.#activeProjects.set(projectPublicId, project) } catch (e) { + this.#l.log('ERROR: could not add project', e) this.#db .delete(projectKeysTable) .where(eq(projectKeysTable.projectId, projectId)) .run() throw e } + this.#l.log('Added project %h, public ID: %S', projectKey, projectPublicId) return projectPublicId } @@ -504,9 +530,15 @@ export class MapeoManager extends TypedEmitter { auth: { localState: authState }, config: { localState: configState }, } = project.$sync.getState() + this.#l.log('Wait for sync, auth state: %o', authState) + this.#l.log('Wait for sync, config state: %o', configState) const isCapabilitySynced = capability !== Capabilities.NO_ROLE_CAPABILITIES const isProjectSettingsSynced = projectSettings !== MapeoProject.EMPTY_PROJECT_SETTINGS + this.#l.log('Wait for sync: %o', { + isCapabilitySynced, + isProjectSettingsSynced, + }) // Assumes every project that someone is invited to has at least one record // in the auth store - the capability record for the invited device const isAuthSynced = authState.want === 0 && authState.have > 0 @@ -516,7 +548,7 @@ export class MapeoManager extends TypedEmitter { const isConfigSynced = configState.want === 0 && configState.have > 0 if ( isCapabilitySynced && - // isProjectSettingsSynced && + isProjectSettingsSynced && isAuthSynced && isConfigSynced ) { @@ -525,6 +557,11 @@ export class MapeoManager extends TypedEmitter { return new Promise((resolve, reject) => { /** @param {import('./sync/sync-state.js').State} syncState */ const onSyncState = (syncState) => { + this.#l.log( + 'Wait for sync: syncState %O\n%O', + syncState.auth, + syncState.config + ) clearTimeout(timeoutId) timeoutId = setTimeout(onTimeout, timeoutMs) if (syncState.auth.dataToSync || syncState.config.dataToSync) return diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 281758d1d..24782d02a 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -34,6 +34,7 @@ import { getDeviceId, projectKeyToId, valueOf } from './utils.js' import { MemberApi } from './member-api.js' import { SyncApi, kSyncReplicate } from './sync/sync-api.js' import Hypercore from 'hypercore' +import { Logger } from './logger.js' /** @typedef {Omit} EditableProjectSettings */ @@ -60,6 +61,7 @@ export class MapeoProject { #syncApi static EMPTY_PROJECT_SETTINGS = EMPTY_PROJECT_SETTINGS + #l /** * @param {Object} opts @@ -72,6 +74,7 @@ export class MapeoProject { * @param {IndexWriter} opts.sharedIndexWriter * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data * @param {import('./local-peers.js').LocalPeers} opts.localPeers + * @param {Logger} [opts.logger] * */ constructor({ @@ -84,7 +87,9 @@ export class MapeoProject { projectSecretKey, encryptionKeys, localPeers, + logger, }) { + this.#l = Logger.create('project', logger) this.#deviceId = getDeviceId(keyManager) this.#projectId = projectKeyToId(projectKey) @@ -114,6 +119,7 @@ export class MapeoProject { keyManager, storage: coreManagerStorage, sqlite, + logger: this.#l, }) const indexWriter = new IndexWriter({ @@ -137,6 +143,7 @@ export class MapeoProject { return doc } }, + logger: this.#l, }) this.#dataStores = { auth: new DataStore({ @@ -251,6 +258,7 @@ export class MapeoProject { this.#syncApi = new SyncApi({ coreManager: this.#coreManager, capabilities: this.#capabilities, + logger: this.#l, }) ///////// 4. Wire up sync @@ -296,6 +304,7 @@ export class MapeoProject { .then(deferred.resolve) .catch(deferred.reject) }) + this.#l.log('Created project instance %h', projectKey) } /** @@ -415,7 +424,8 @@ export class MapeoProject { return extractEditableProjectSettings( await this.#dataTypes.projectSettings.getByDocId(this.#projectId) ) - } catch { + } catch (e) { + this.#l.log('No project settings') return /** @type {EditableProjectSettings} */ (EMPTY_PROJECT_SETTINGS) } } diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index f91a0e204..a9356a631 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -1,5 +1,6 @@ import mapObject from 'map-obj' import { NAMESPACES } from '../core-manager/index.js' +import { Logger } from '../logger.js' /** * @typedef {import('../core-manager/index.js').Namespace} Namespace @@ -29,6 +30,7 @@ export class PeerSyncController { #downloadingRanges = new Map() /** @type {SyncStatus} */ #prevSyncStatus = createNamespaceMap('unknown') + #log /** * @param {object} opts @@ -36,15 +38,24 @@ export class PeerSyncController { * @param {import("../core-manager/index.js").CoreManager} opts.coreManager * @param {import("./sync-state.js").SyncState} opts.syncState * @param {import("../capabilities.js").Capabilities} opts.capabilities + * @param {Logger} [opts.logger] */ - constructor({ protomux, coreManager, syncState, capabilities }) { + constructor({ protomux, coreManager, syncState, capabilities, logger }) { + // @ts-ignore + this.#log = (formatter, ...args) => { + const log = Logger.create('peer', logger).log + return log.apply(null, [ + `[%h] ${formatter}`, + protomux.stream.remotePublicKey, + ...args, + ]) + } this.#coreManager = coreManager this.#protomux = protomux this.#capabilities = capabilities // Always need to replicate the project creator core - coreManager.creatorCore.replicate(protomux) - this.#replicatingCores.add(coreManager.creatorCore) + this.#replicateCore(coreManager.creatorCore) coreManager.on('add-core', this.#handleAddCore) syncState.on('state', this.#handleStateChange) @@ -52,6 +63,14 @@ export class PeerSyncController { this.#updateEnabledNamespaces() } + get peerKey() { + return this.#protomux.stream.remotePublicKey + } + + get peerId() { + return this.peerKey?.toString('hex') + } + /** * Enable syncing of data (in the data and blob namespaces) */ @@ -78,6 +97,7 @@ export class PeerSyncController { * @param {import("../core-manager/core-index.js").CoreRecord} coreRecord */ #handleAddCore = ({ core, namespace }) => { + this.#log('Add core %h to %s', core.key, namespace) if (!this.#enabledNamespaces.has(namespace)) return this.#replicateCore(core) } @@ -93,12 +113,12 @@ export class PeerSyncController { // connected. We shouldn't get a state change before the noise stream has // connected, but if we do we can ignore it because we won't have any useful // information until it connects. - if (!this.#protomux.stream.remotePublicKey) return - const peerId = this.#protomux.stream.remotePublicKey.toString('hex') - this.#syncStatus = getSyncStatus(peerId, state) + if (!this.peerId) return + this.#syncStatus = getSyncStatus(this.peerId, state) const localState = mapObject(state, (ns, nsState) => { return [ns, nsState.localState] }) + this.#log('state %O', state) // Map of which namespaces have received new data since last sync change const didUpdate = mapObject(state, (ns) => { @@ -118,16 +138,15 @@ export class PeerSyncController { if (didUpdate.auth) { try { - const cap = await this.#capabilities.getCapabilities(peerId) + const cap = await this.#capabilities.getCapabilities(this.peerId) this.#syncCapability = cap.sync } catch (e) { - // Any error, consider sync blocked - this.#syncCapability = createNamespaceMap('blocked') + this.#log('Error reading capability', e) + // Any error, consider sync unknown + this.#syncCapability = createNamespaceMap('unknown') } } - // console.log(peerId.slice(0, 7), this.#syncCapability) - // console.log(peerId.slice(0, 7), didUpdate) - // console.dir(state, { depth: null, colors: true }) + this.#log('capability %o', this.#syncCapability) // If any namespace has new data, update what is enabled if (Object.values(didUpdate).indexOf(true) > -1) { @@ -177,7 +196,12 @@ export class PeerSyncController { */ #replicateCore(core) { if (this.#replicatingCores.has(core)) return + this.#log('replicating core %k', core.key) core.replicate(this.#protomux) + core.on('peer-remove', (peer) => { + if (!peer.remotePublicKey.equals(this.peerKey)) return + this.#log('peer-remove %h from core %k', peer.remotePublicKey, core.key) + }) this.#replicatingCores.add(core) } @@ -190,6 +214,7 @@ export class PeerSyncController { (peer) => peer.protomux === this.#protomux ) if (!peerToUnreplicate) return + this.#log('unreplicating core %k', core.key) peerToUnreplicate.channel.close() this.#replicatingCores.delete(core) } @@ -203,6 +228,7 @@ export class PeerSyncController { this.#replicateCore(core) } this.#enabledNamespaces.add(namespace) + this.#log('enabled namespace %s', namespace) } /** @@ -214,6 +240,7 @@ export class PeerSyncController { this.#unreplicateCore(core) } this.#enabledNamespaces.delete(namespace) + this.#log('disabled namespace %s', namespace) } } diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index 9508adc13..6fe8dff85 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -1,6 +1,7 @@ import { TypedEmitter } from 'tiny-typed-emitter' import { SyncState } from './sync-state.js' import { PeerSyncController } from './peer-sync-controller.js' +import { Logger } from '../logger.js' export const kSyncReplicate = Symbol('replicate sync') @@ -20,6 +21,7 @@ export class SyncApi extends TypedEmitter { #peerSyncControllers = new Map() /** @type {Set<'local' | 'remote'>} */ #dataSyncEnabled = new Set() + #l /** * @@ -27,9 +29,11 @@ export class SyncApi extends TypedEmitter { * @param {import('../core-manager/index.js').CoreManager} opts.coreManager * @param {import("../capabilities.js").Capabilities} opts.capabilities * @param {number} [opts.throttleMs] + * @param {Logger} [opts.logger] */ - constructor({ coreManager, throttleMs = 200, capabilities }) { + constructor({ coreManager, throttleMs = 200, capabilities, logger }) { super() + this.#l = Logger.create('syncApi', logger) this.#coreManager = coreManager this.#capabilities = capabilities this.syncState = new SyncState({ coreManager, throttleMs }) @@ -46,6 +50,7 @@ export class SyncApi extends TypedEmitter { start() { if (this.#dataSyncEnabled.has('local')) return this.#dataSyncEnabled.add('local') + this.#l.log('Starting data sync') for (const peerSyncController of this.#peerSyncControllers.values()) { peerSyncController.enableDataSync() } @@ -57,6 +62,7 @@ export class SyncApi extends TypedEmitter { stop() { if (!this.#dataSyncEnabled.has('local')) return this.#dataSyncEnabled.delete('local') + this.#l.log('Stopping data sync') for (const peerSyncController of this.#peerSyncControllers.values()) { peerSyncController.disableDataSync() } @@ -66,13 +72,20 @@ export class SyncApi extends TypedEmitter { * @param {import('protomux')} protomux A protomux instance */ [kSyncReplicate](protomux) { - if (this.#peerSyncControllers.has(protomux)) return + if (this.#peerSyncControllers.has(protomux)) { + this.#l.log( + 'Existing sync controller for peer %h', + protomux.stream.remotePublicKey + ) + return + } const peerSyncController = new PeerSyncController({ protomux, coreManager: this.#coreManager, syncState: this.syncState, capabilities: this.#capabilities, + logger: this.#l, }) if (this.#dataSyncEnabled.has('local')) { peerSyncController.enableDataSync() From 2fb10727a7d8b327cf77f75834c90fe4654dc662 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 16:57:30 +0900 Subject: [PATCH 24/69] fix timeout --- src/mapeo-manager.js | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 1e65733d0..235c8baa9 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -530,15 +530,9 @@ export class MapeoManager extends TypedEmitter { auth: { localState: authState }, config: { localState: configState }, } = project.$sync.getState() - this.#l.log('Wait for sync, auth state: %o', authState) - this.#l.log('Wait for sync, config state: %o', configState) const isCapabilitySynced = capability !== Capabilities.NO_ROLE_CAPABILITIES const isProjectSettingsSynced = projectSettings !== MapeoProject.EMPTY_PROJECT_SETTINGS - this.#l.log('Wait for sync: %o', { - isCapabilitySynced, - isProjectSettingsSynced, - }) // Assumes every project that someone is invited to has at least one record // in the auth store - the capability record for the invited device const isAuthSynced = authState.want === 0 && authState.have > 0 @@ -557,14 +551,11 @@ export class MapeoManager extends TypedEmitter { return new Promise((resolve, reject) => { /** @param {import('./sync/sync-state.js').State} syncState */ const onSyncState = (syncState) => { - this.#l.log( - 'Wait for sync: syncState %O\n%O', - syncState.auth, - syncState.config - ) clearTimeout(timeoutId) - timeoutId = setTimeout(onTimeout, timeoutMs) - if (syncState.auth.dataToSync || syncState.config.dataToSync) return + if (syncState.auth.dataToSync || syncState.config.dataToSync) { + timeoutId = setTimeout(onTimeout, timeoutMs) + return + } project.$sync.off('sync-state', onSyncState) resolve(this.#waitForInitialSync(project, { timeoutMs })) } From b5f1a0017de37126f5cfb840746cd5c191e9ff00 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 16:58:14 +0900 Subject: [PATCH 25/69] fix: Add new cores to the indexer (!!!) This caused a day of work: a bug from months back --- src/datastore/index.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/datastore/index.js b/src/datastore/index.js index 4655c0c06..5654927c7 100644 --- a/src/datastore/index.js +++ b/src/datastore/index.js @@ -66,6 +66,10 @@ export class DataStore extends TypedEmitter { storage, batch: (entries) => this.#handleEntries(entries), }) + coreManager.on('add-core', (coreRecord) => { + if (coreRecord.namespace !== namespace) return + this.#coreIndexer.addCore(coreRecord.core) + }) // Forward events from coreIndexer this.on('newListener', (eventName, listener) => { From d3fb8ad4c91246345a5b980894b1513bb26ad033 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 16:58:35 +0900 Subject: [PATCH 26/69] remove unnecessary log stmt --- src/sync/peer-sync-controller.js | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index a9356a631..927fd2b9f 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -97,7 +97,6 @@ export class PeerSyncController { * @param {import("../core-manager/core-index.js").CoreRecord} coreRecord */ #handleAddCore = ({ core, namespace }) => { - this.#log('Add core %h to %s', core.key, namespace) if (!this.#enabledNamespaces.has(namespace)) return this.#replicateCore(core) } From 32c4d32a25f01d0b2c333a3114a91c9f8515adcc Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 17:23:51 +0900 Subject: [PATCH 27/69] get capabilities.getMany() to include creator --- src/capabilities.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/capabilities.js b/src/capabilities.js index 666b8687e..8b683c95c 100644 --- a/src/capabilities.js +++ b/src/capabilities.js @@ -211,17 +211,21 @@ export class Capabilities { */ async getAll() { const roles = await this.#dataType.getMany() + /** @type {Record} */ + const capabilities = {} let projectCreatorDeviceId try { projectCreatorDeviceId = await this.#coreOwnership.getOwner( this.#projectCreatorAuthCoreId ) + // Default to creator capabilities, but can be overwritten if a different + // role is set below + capabilities[projectCreatorDeviceId] = CREATOR_CAPABILITIES } catch (e) { // Not found, we don't know who the project creator is so we can't include // them in the returned map } - /** @type {Record} */ - const capabilities = {} + for (const role of roles) { const deviceId = role.docId if (!isKnownRoleId(role.roleId)) continue From fe01caa162df99f8204c7b0ad4db10a53ecc986a Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 17:24:01 +0900 Subject: [PATCH 28/69] fix invite test --- test-e2e/manager-invite.js | 226 ++++++++++++++++++------------------- 1 file changed, 109 insertions(+), 117 deletions(-) diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index 4a0b866ce..a6dcb8ac3 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -1,166 +1,101 @@ import { test } from 'brittle' -import { KeyManager } from '@mapeo/crypto' -import pDefer from 'p-defer' import RAM from 'random-access-memory' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' -import { MapeoManager, kRPC } from '../src/mapeo-manager.js' -import { replicate } from '../tests/helpers/local-peers.js' +import { MapeoManager, kManagerReplicate } from '../src/mapeo-manager.js' +import { once } from 'node:events' +import sodium from 'sodium-universal' test('member invite accepted', async (t) => { - t.plan(10) - - const deferred = pDefer() - - const creator = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) - + const creator = createManager('creator') await creator.setDeviceInfo({ name: 'Creator' }) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) - creator[kRPC].on('peers', async (peers) => { - t.is(peers.length, 1) - - const response = await creatorProject.$member.invite(peers[0].deviceId, { - roleId: MEMBER_ROLE_ID, - }) - - t.is(response, InviteResponse_Decision.ACCEPT) - - deferred.resolve() - }) - - /** @type {string | undefined} */ - let expectedInvitorPeerId - - const joiner = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) + const joiner = createManager('joiner1') await joiner.setDeviceInfo({ name: 'Joiner' }) - t.exception( + await t.exception( async () => joiner.getProject(createdProjectId), 'joiner cannot get project instance before being invited and added to project' ) - joiner[kRPC].on('peers', (peers) => { - t.is(peers.length, 1) - expectedInvitorPeerId = peers[0].deviceId - }) + const destroy = replicate(creator, joiner) - joiner.invite.on('invite-received', async (invite) => { - t.is(invite.projectId, createdProjectId) - t.is(invite.peerId, expectedInvitorPeerId) - t.is(invite.projectName, 'Mapeo') - // TODO: Check role being invited for (needs https://github.com/digidem/mapeo-core-next/issues/275) - - await joiner.invite.accept(invite.projectId) + const responsePromise = creatorProject.$member.invite(joiner.deviceId, { + roleId: MEMBER_ROLE_ID, }) + const [invite] = await once(joiner.invite, 'invite-received') + t.is(invite.projectId, createdProjectId, 'projectId of invite matches') + t.is(invite.peerId, creator.deviceId, 'deviceId of invite matches') + t.is(invite.projectName, 'Mapeo', 'project name of invite matches') - replicate(creator[kRPC], joiner[kRPC]) + await joiner.invite.accept(invite.projectId) - await deferred.promise + t.is( + await responsePromise, + InviteResponse_Decision.ACCEPT, + 'correct invite response' + ) /// After invite flow has completed... - const joinerListedProjects = await joiner.listProjects() - - t.is(joinerListedProjects.length, 1, 'project added to joiner') t.alike( - joinerListedProjects[0], - { - name: 'Mapeo', - projectId: createdProjectId, - createdAt: undefined, - updatedAt: undefined, - }, + await joiner.listProjects(), + await creator.listProjects(), 'project info recorded in joiner successfully' ) - const joinerProject = await joiner.getProject( - joinerListedProjects[0].projectId - ) + const joinerProject = await joiner.getProject(createdProjectId) - t.ok(joinerProject, 'can create joiner project instance') + t.alike( + await joinerProject.$getProjectSettings(), + await creatorProject.$getProjectSettings(), + 'Project settings match' + ) - // TODO: Get project settings of joiner and ensure they're similar to creator's project's settings - // const joinerProjectSettings = await joinerProject.$getProjectSettings() - // t.alike(joinerProjectSettings, { defaultPresets: undefined, name: 'Mapeo' }) + t.alike( + await creatorProject.$member.getMany(), + await joinerProject.$member.getMany(), + 'Project members match' + ) - // TODO: Get members of creator project and assert info matches joiner - // const creatorProjectMembers = await creatorProject.$member.getMany() - // t.is(creatorProjectMembers.length, 1) - // t.alike(creatorProjectMembers[0], await joiner.getDeviceInfo()) + await destroy() }) test('member invite rejected', async (t) => { - t.plan(9) - - const deferred = pDefer() - - const creator = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) - + const creator = createManager('creator') await creator.setDeviceInfo({ name: 'Creator' }) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) - creator[kRPC].on('peers', async (peers) => { - t.is(peers.length, 1) - - const response = await creatorProject.$member.invite(peers[0].deviceId, { - roleId: MEMBER_ROLE_ID, - }) - - t.is(response, InviteResponse_Decision.REJECT) - - deferred.resolve() - }) - - /** @type {string | undefined} */ - let expectedInvitorPeerId - - const joiner = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) - + const joiner = createManager('joiner1') await joiner.setDeviceInfo({ name: 'Joiner' }) - t.exception( + await t.exception( async () => joiner.getProject(createdProjectId), 'joiner cannot get project instance before being invited and added to project' ) - joiner[kRPC].on('peers', (peers) => { - t.is(peers.length, 1) - expectedInvitorPeerId = peers[0].deviceId - }) - - joiner.invite.on('invite-received', async (invite) => { - t.is(invite.projectId, createdProjectId) - t.is(invite.peerId, expectedInvitorPeerId) - t.is(invite.projectName, 'Mapeo') - // TODO: Check role being invited for (needs https://github.com/digidem/mapeo-core-next/issues/275) + const destroy = replicate(creator, joiner) - await joiner.invite.reject(invite.projectId) + const responsePromise = creatorProject.$member.invite(joiner.deviceId, { + roleId: MEMBER_ROLE_ID, }) + const [invite] = await once(joiner.invite, 'invite-received') + t.is(invite.projectId, createdProjectId, 'projectId of invite matches') + t.is(invite.peerId, creator.deviceId, 'deviceId of invite matches') + t.is(invite.projectName, 'Mapeo', 'project name of invite matches') - replicate(creator[kRPC], joiner[kRPC]) + await joiner.invite.reject(invite.projectId) - await deferred.promise + t.is( + await responsePromise, + InviteResponse_Decision.REJECT, + 'correct invite response' + ) /// After invite flow has completed... @@ -173,7 +108,64 @@ test('member invite rejected', async (t) => { 'joiner cannot get project instance' ) - // TODO: Get members of creator project and assert joiner not added - // const creatorProjectMembers = await creatorProject.$member.getMany() - // t.is(creatorProjectMembers.length, 0) + t.is( + (await creatorProject.$member.getMany()).length, + 1, + 'Only 1 member in project still' + ) + + await destroy() }) + +/** + * @param {MapeoManager} mm1 + * @param {MapeoManager} mm2 + */ +export function replicate(mm1, mm2) { + const r1 = mm1[kManagerReplicate](true) + const r2 = mm2[kManagerReplicate](false) + + r1.pipe(r2).pipe(r1) + + /** @param {Error} [e] */ + return async function destroy(e) { + return Promise.all([ + /** @type {Promise} */ + ( + new Promise((res) => { + r1.on('close', res) + r1.destroy(e) + }) + ), + /** @type {Promise} */ + ( + new Promise((res) => { + r2.on('close', res) + r2.destroy(e) + }) + ), + ]) + } +} + +/** @param {string} [seed] */ +function createManager(seed) { + return new MapeoManager({ + rootKey: getRootKey(seed), + dbFolder: ':memory:', + coreStorage: () => new RAM(), + }) +} + +/** @param {string} [seed] */ +function getRootKey(seed) { + const key = Buffer.allocUnsafe(16) + if (!seed) { + sodium.randombytes_buf(key) + } else { + const seedBuf = Buffer.alloc(32) + sodium.crypto_generichash(seedBuf, Buffer.from(seed)) + sodium.randombytes_buf_deterministic(key, seedBuf) + } + return key +} From 5a47b569b6ad7b417fb866f2c518626fca9b39ad Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 30 Oct 2023 18:01:40 +0900 Subject: [PATCH 29/69] keep blob cores sparse --- src/core-manager/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index 733bb6d2a..cebe2e767 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -279,7 +279,7 @@ export class CoreManager extends TypedEmitter { keyPair, encryptionKey: this.#encryptionKeys[namespace], // Starts live download of core immediately - sparse: false, + sparse: namespace === 'blob', }) // @ts-ignore - ensure key is defined before hypercore is ready core.key = key From 9921f66f666f3091c145670ed69c66e9d235db0f Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 7 Nov 2023 15:38:27 +0900 Subject: [PATCH 30/69] optional param for LocalPeers --- src/local-peers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index aa8df597e..14424e63e 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -207,10 +207,10 @@ export class LocalPeers extends TypedEmitter { /** * - * @param {object} opts + * @param {object} [opts] * @param {Logger} [opts.logger] */ - constructor({ logger }) { + constructor({ logger } = {}) { super() this.#l = Logger.create('localPeers', logger) } From 68455be7feac252609f12b6e81207fe2f2a8defb Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 10 Nov 2023 17:01:09 +0900 Subject: [PATCH 31/69] re-org sync and replication Removes old replication code attached to CoreManager Still needs tests to be updated --- src/core-manager/index.js | 180 ++---------------- src/core-manager/replication-state-machine.js | 76 -------- src/datastore/index.js | 12 +- src/mapeo-project.js | 35 ++-- src/sync/peer-sync-controller.js | 25 +++ src/sync/sync-api.js | 83 +++++++- tests/blob-server.js | 2 +- tests/blob-store/blob-store.js | 8 +- 8 files changed, 151 insertions(+), 270 deletions(-) delete mode 100644 src/core-manager/replication-state-machine.js diff --git a/src/core-manager/index.js b/src/core-manager/index.js index cebe2e767..3a31e76d6 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -2,11 +2,8 @@ import { TypedEmitter } from 'tiny-typed-emitter' import Corestore from 'corestore' import assert from 'node:assert' -import { once } from 'node:events' -import Hypercore from 'hypercore' import { HaveExtension, ProjectExtension } from '../generated/extensions.js' import { CoreIndex } from './core-index.js' -import { ReplicationStateMachine } from './replication-state-machine.js' import * as rle from './bitfield-rle.js' import { Logger } from '../logger.js' @@ -30,7 +27,6 @@ const CREATE_SQL = `CREATE TABLE IF NOT EXISTS ${TABLE} ( /** @typedef {(typeof NAMESPACES)[number]} Namespace */ /** @typedef {{ core: Core, key: Buffer, namespace: Namespace }} CoreRecord */ /** @typedef {import('streamx').Duplex} DuplexStream */ -/** @typedef {{ rsm: ReplicationStateMachine, stream: DuplexStream, cores: Set }} ReplicationRecord */ /** * @typedef {Object} Events * @property {(coreRecord: CoreRecord) => void} add-core @@ -48,8 +44,6 @@ export class CoreManager extends TypedEmitter { #projectKey #addCoreSqlStmt #encryptionKeys - /** @type {Set} */ - #replications = new Set() #projectExtension /** @type {'opened' | 'closing' | 'closed'} */ #state = 'opened' @@ -222,13 +216,13 @@ export class CoreManager extends TypedEmitter { * Get a core by its discovery key * * @param {Buffer} discoveryKey - * @returns {Core | undefined} + * @returns {CoreRecord | undefined} */ getCoreByDiscoveryKey(discoveryKey) { const coreRecord = this.#coreIndex.getByDiscoveryId( discoveryKey.toString('hex') ) - return coreRecord && coreRecord.core + return coreRecord } /** @@ -241,10 +235,6 @@ export class CoreManager extends TypedEmitter { for (const { core } of this.#coreIndex) { promises.push(core.close()) } - for (const { stream } of this.#replications) { - promises.push(once(stream, 'close')) - stream.destroy() - } await Promise.all(promises) this.#state = 'closed' } @@ -310,13 +300,6 @@ export class CoreManager extends TypedEmitter { }) } - for (const { stream, rsm, cores } of this.#replications.values()) { - if (cores.has(core)) continue - if (rsm.state.enabledNamespaces.has(namespace)) { - core.replicate(stream) - } - } - if (persist) { this.#addCoreSqlStmt.run({ publicKey: key, namespace }) } @@ -332,148 +315,39 @@ export class CoreManager extends TypedEmitter { } /** - * Start replicating cores managed by CoreManager to a Noise Secret Stream (as - * created by @hyperswarm/secret-stream). Important: only one CoreManager can - * be replicated to a given stream - attempting to replicate a second - * CoreManager to the same stream will cause sharing of auth core keys to - * fail - see https://github.com/holepunchto/corestore/issues/45 + * Send an extension message over the project creator core replication stream + * requesting a core key for the given discovery key. * - * Initially only cores in the `auth` namespace are replicated to the stream. - * All cores in the `auth` namespace are shared to all peers who have the - * `rootCoreKey` core, and also replicated to the stream - * - * To start replicating other namespaces call `enableNamespace(ns)` on the - * returned state machine - * - * @param {import('../types.js').NoiseStream | import('../types.js').ProtocolStream} noiseStream framed noise secret stream, i.e. @hyperswarm/secret-stream - */ - replicate(noiseStream) { - if (this.#state !== 'opened') throw new Error('Core manager is closed') - if ( - /** @type {import('../types.js').ProtocolStream} */ (noiseStream) - .noiseStream?.userData - ) { - console.warn( - 'Passed an existing protocol stream to coreManager.replicate(). Other corestores and core managers replicated to this stream will no longer automatically inject shared cores into the stream' - ) - } - // @ts-expect-error - too complex to type right now - const stream = Hypercore.createProtocolStream(noiseStream) - const protocol = stream.noiseStream.userData - if (!protocol) throw new Error('Invalid stream') - // If the noise stream already had a protomux instance attached to - // noiseStream.userData, then Hypercore.createProtocolStream does not attach - // the ondiscoverykey listener, so we make sure we are listening for this, - // and that we override any previous notifier that was attached to protomux. - // This means that only one Core Manager instance can currently be - // replicated to a stream if we want sharing of unknown auth cores to work. - protocol.pair( - { protocol: 'hypercore/alpha' }, - /** @param {Buffer} discoveryKey */ async (discoveryKey) => { - this.handleDiscoveryKey(discoveryKey, stream) - } - ) - - /** @type {ReplicationRecord['cores']} */ - const replicatingCores = new Set() - const rsm = new ReplicationStateMachine({ - enableNamespace: (namespace) => { - for (const { core } of this.getCores(namespace)) { - if (replicatingCores.has(core)) continue - core.replicate(protocol) - replicatingCores.add(core) - } - }, - disableNamespace: (namespace) => { - for (const { core } of this.getCores(namespace)) { - if (core === this.creatorCore) continue - unreplicate(core, protocol) - replicatingCores.delete(core) - } - }, - }) - - // Always need to replicate the project creator core - this.creatorCore.replicate(protocol) - replicatingCores.add(this.creatorCore) - - // For now enable auth namespace here, rather than in sync controller - rsm.enableNamespace('auth') - - /** @type {ReplicationRecord} */ - const replicationRecord = { stream, rsm, cores: replicatingCores } - this.#replications.add(replicationRecord) - - stream.once('close', () => { - rsm.disableAll() - rsm.removeAllListeners() - this.#replications.delete(replicationRecord) - }) - - return rsm - } - - /** + * @param {Buffer} peerKey * @param {Buffer} discoveryKey - * @param {any} stream */ - async handleDiscoveryKey(discoveryKey, stream) { - const discoveryId = discoveryKey.toString('hex') - const peer = await this.#findPeer(stream.remotePublicKey) + requestCoreKey(peerKey, discoveryKey) { + // No-op if we already have this core + if (this.getCoreByDiscoveryKey(discoveryKey)) return + const peer = this.#creatorCore.peers.find((peer) => { + return peer.remotePublicKey.equals(peerKey) + }) if (!peer) { + // This should not happen because this is only called from SyncApi, which + // checks the peer exists before calling this method. this.#l.log( - 'Receive dk %h but no connected peer %h', + 'Attempted to request core key for %h, but no connected peer %h', discoveryKey, - stream.remotePublicKey + peerKey ) - // TODO: How to handle this and when does it happen? - return - } - // If we already know about this core, then we will add it to the - // replication stream when we are ready - if (this.#coreIndex.getByDiscoveryId(discoveryId)) { - this.#l.log('Received dk %h, but already have core', discoveryKey) return } - this.#l.log('Requesting core key for dk %h', discoveryKey) + this.#l.log( + 'Requesting core key for discovery key %h from peer %h', + discoveryKey, + peerKey + ) const message = ProjectExtension.fromPartial({ wantCoreKeys: [discoveryKey], }) this.#projectExtension.send(message, peer) } - /** - * @param {Buffer} publicKey - * @param {{ timeout?: number }} [opts] - */ - async #findPeer(publicKey, { timeout = 200 } = {}) { - const creatorCore = this.#creatorCore - const peer = creatorCore.peers.find((peer) => { - return peer.remotePublicKey.equals(publicKey) - }) - if (peer) return peer - // This is called from the from the handleDiscoveryId event, which can - // happen before the peer connection is fully established, so we wait for - // the `peer-add` event, with a timeout in case the peer never gets added - return new Promise(function (res) { - const timeoutId = setTimeout(function () { - creatorCore.off('peer-add', onPeer) - res(null) - }, timeout) - - creatorCore.on('peer-add', onPeer) - - /** @param {any} peer */ - function onPeer(peer) { - if (peer.remotePublicKey.equals(publicKey)) { - clearTimeout(timeoutId) - creatorCore.off('peer-add', onPeer) - res(peer) - } - } - }) - } - /** * @param {ProjectExtension} msg * @param {any} peer @@ -593,17 +467,3 @@ const HaveExtensionCodec = { } }, } - -/** - * - * @param {Hypercore<'binary', any>} core - * @param {import('protomux')} protomux - */ -export function unreplicate(core, protomux) { - const peerToUnreplicate = core.peers.find( - (peer) => peer.protomux === protomux - ) - if (!peerToUnreplicate) return - peerToUnreplicate.channel.close() - return -} diff --git a/src/core-manager/replication-state-machine.js b/src/core-manager/replication-state-machine.js deleted file mode 100644 index a71dfe61d..000000000 --- a/src/core-manager/replication-state-machine.js +++ /dev/null @@ -1,76 +0,0 @@ -import { TypedEmitter } from 'tiny-typed-emitter' - -/** @typedef {import('./index.js').Namespace} Namespace */ -/** @typedef {Set} EnabledNamespaces */ -/** @typedef {{ enabledNamespaces: EnabledNamespaces }} ReplicationState */ - -/** - * @typedef {object} StateMachineEvents - * @property {(state: ReplicationState) => void } state - */ - -/** - * A simple state machine to manage which namespaces are enabled for replication - * - * @extends {TypedEmitter} - */ -export class ReplicationStateMachine extends TypedEmitter { - /** @type {ReplicationState} */ - #state = { - enabledNamespaces: new Set(), - } - #enableNamespace - #disableNamespace - - /** - * - * @param {object} opts - * @param {(namespace: Namespace) => void} opts.enableNamespace - * @param {(namespace: Namespace) => void} opts.disableNamespace - */ - constructor({ enableNamespace, disableNamespace }) { - super() - this.#enableNamespace = enableNamespace - this.#disableNamespace = disableNamespace - } - - get state() { - return this.#state - } - - /** - * Enable a namespace for replication - will add known cores in the namespace - * to the replication stream - * - * @param {Namespace} namespace */ - enableNamespace(namespace) { - if (this.#state.enabledNamespaces.has(namespace)) return - this.#state.enabledNamespaces.add(namespace) - this.#enableNamespace(namespace) - this.emit('state', this.#state) - } - - /** - * Disable a namespace for replication - will remove cores in the namespace - * from the replication stream - * - * @param {Namespace} namespace - */ - disableNamespace(namespace) { - if (!this.#state.enabledNamespaces.has(namespace)) return - this.#state.enabledNamespaces.delete(namespace) - this.#disableNamespace(namespace) - this.emit('state', this.#state) - } - - /** - * @internal - * Should only be called when the stream is closed, because no obvious way to - * implement this otherwise. - */ - disableAll() { - if (!this.#state.enabledNamespaces.size) return - this.#state.enabledNamespaces.clear() - this.emit('state', this.#state) - } -} diff --git a/src/datastore/index.js b/src/datastore/index.js index 5654927c7..be98ffcf1 100644 --- a/src/datastore/index.js +++ b/src/datastore/index.js @@ -168,9 +168,9 @@ export class DataStore extends TypedEmitter { */ async read(versionId) { const { coreDiscoveryKey, index } = parseVersionId(versionId) - const core = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey) - if (!core) throw new Error('Invalid versionId') - const block = await core.get(index, { wait: false }) + const coreRecord = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey) + if (!coreRecord) throw new Error('Invalid versionId') + const block = await coreRecord.core.get(index, { wait: false }) if (!block) throw new Error('Not Found') return decode(block, { coreDiscoveryKey, index }) } @@ -190,9 +190,9 @@ export class DataStore extends TypedEmitter { /** @param {string} versionId */ async readRaw(versionId) { const { coreDiscoveryKey, index } = parseVersionId(versionId) - const core = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey) - if (!core) throw new Error('core not found') - const block = await core.get(index, { wait: false }) + const coreRecord = this.#coreManager.getCoreByDiscoveryKey(coreDiscoveryKey) + if (!coreRecord) throw new Error('core not found') + const block = await coreRecord.core.get(index, { wait: false }) if (!block) throw new Error('Not Found') return block } diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 24782d02a..42637c629 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -32,8 +32,7 @@ import { import { Capabilities } from './capabilities.js' import { getDeviceId, projectKeyToId, valueOf } from './utils.js' import { MemberApi } from './member-api.js' -import { SyncApi, kSyncReplicate } from './sync/sync-api.js' -import Hypercore from 'hypercore' +import { SyncApi, kHandleDiscoveryKey } from './sync/sync-api.js' import { Logger } from './logger.js' /** @typedef {Omit} EditableProjectSettings */ @@ -261,26 +260,25 @@ export class MapeoProject { logger: this.#l, }) - ///////// 4. Wire up sync + ///////// 4. Replicate local peers automatically // Replicate already connected local peers for (const peer of localPeers.peers) { if (peer.status !== 'connected') continue - this.#syncApi[kSyncReplicate](peer.protomux) + this.#coreManager.creatorCore.replicate(peer.protomux) } - localPeers.on('discovery-key', (discoveryKey, stream) => { - // The core identified by this discovery key might not be part of this - // project, but we can't know that so we will request it from the peer if - // we don't have it. The peer will not return the core key unless it _is_ - // part of this project - this.#coreManager.handleDiscoveryKey(discoveryKey, stream) - }) - // When a new peer is found, try to replicate (if it is not a member of the // project it will fail the capability check and be ignored) localPeers.on('peer-add', (peer) => { - this.#syncApi[kSyncReplicate](peer.protomux) + this.#coreManager.creatorCore.replicate(peer.protomux) + }) + + // This happens whenever a peer replicates a core to the stream. SyncApi + // handles replicating this core if we also have it, or requesting the key + // for the core. + localPeers.on('discovery-key', (discoveryKey, stream) => { + this.#syncApi[kHandleDiscoveryKey](discoveryKey, stream) }) ///////// 5. Write core ownership record @@ -440,18 +438,17 @@ export class MapeoProject { * and only this project will replicate (to replicate multiple projects you * need to replicate the manager instance via manager[kManagerReplicate]) * - * @param {Exclude[0], boolean>} stream A duplex stream, a @hyperswarm/secret-stream, or a Protomux instance + * @param {Parameters[0]} stream A duplex stream, a @hyperswarm/secret-stream, or a Protomux instance * @returns */ [kProjectReplicate](stream) { - const replicationStream = Hypercore.createProtocolStream(stream, { + // @ts-expect-error - hypercore types need updating + const replicationStream = this.#coreManager.creatorCore.replicate(stream, { + // @ts-ignore - hypercore types do not currently include this option ondiscoverykey: async (discoveryKey) => { - this.#coreManager.handleDiscoveryKey(discoveryKey, replicationStream) + this.#syncApi[kHandleDiscoveryKey](discoveryKey, replicationStream) }, }) - const protomux = replicationStream.noiseStream.userData - // @ts-ignore - got fed up jumping through hoops to keep TS heppy - this.#syncApi[kSyncReplicate](protomux) return replicationStream } diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index 927fd2b9f..c076a720a 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -90,6 +90,31 @@ export class PeerSyncController { this.#updateEnabledNamespaces() } + /** + * @param {Buffer} discoveryKey + */ + handleDiscoveryKey(discoveryKey) { + const coreRecord = this.#coreManager.getCoreByDiscoveryKey(discoveryKey) + // If we already know about this core, then we will add it to the + // replication stream when we are ready + if (coreRecord) { + this.#log( + 'Received discovery key %h, but already have core in namespace %s', + discoveryKey, + coreRecord.namespace + ) + if (this.#enabledNamespaces.has(coreRecord.namespace)) { + this.#replicateCore(coreRecord.core) + } + return + } + if (!this.peerKey) { + this.#log('Unexpected null peerKey') + return + } + this.#coreManager.requestCoreKey(this.peerKey, discoveryKey) + } + /** * Handler for 'core-add' event from CoreManager * Bound to `this` (defined as static property) diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index 6fe8dff85..45629450a 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -3,7 +3,7 @@ import { SyncState } from './sync-state.js' import { PeerSyncController } from './peer-sync-controller.js' import { Logger } from '../logger.js' -export const kSyncReplicate = Symbol('replicate sync') +export const kHandleDiscoveryKey = Symbol('handle discovery key') /** * @typedef {object} SyncEvents @@ -21,6 +21,8 @@ export class SyncApi extends TypedEmitter { #peerSyncControllers = new Map() /** @type {Set<'local' | 'remote'>} */ #dataSyncEnabled = new Set() + /** @type {Map>} */ + #pendingDiscoveryKeys = new Map() #l /** @@ -38,6 +40,38 @@ export class SyncApi extends TypedEmitter { this.#capabilities = capabilities this.syncState = new SyncState({ coreManager, throttleMs }) this.syncState.on('state', this.emit.bind(this, 'sync-state')) + + this.#coreManager.creatorCore.on('peer-add', this.#handlePeerAdd) + this.#coreManager.creatorCore.on('peer-remove', this.#handlePeerRemove) + } + + /** @type {import('../local-peers.js').LocalPeersEvents['discovery-key']} */ + [kHandleDiscoveryKey](discoveryKey, stream) { + const protomux = stream.noiseStream.userData + const peerSyncController = this.#peerSyncControllers.get(protomux) + if (peerSyncController) { + peerSyncController.handleDiscoveryKey(discoveryKey) + return + } + // We will reach here if we are not part of the project, so we can ignore + // these keys. However it's also possible to reach here when we are part of + // a project, but the creator core `peer-add` event has not yet fired, so we + // queue this to be handled in `#handlePeerAdd` + const peerQueue = this.#pendingDiscoveryKeys.get(protomux) || new Set() + peerQueue.add(discoveryKey) + this.#pendingDiscoveryKeys.set(protomux, peerQueue) + + // If we _are_ part of the project, the `peer-add` should happen very soon + // after we get a discovery-key event, so we cleanup our queue to avoid + // memory leaks for any discovery keys that have not been handled. + setTimeout(() => { + const peerQueue = this.#pendingDiscoveryKeys.get(protomux) + if (!peerQueue) return + peerQueue.delete(discoveryKey) + if (peerQueue.size === 0) { + this.#pendingDiscoveryKeys.delete(protomux) + } + }, 500) } getState() { @@ -69,17 +103,24 @@ export class SyncApi extends TypedEmitter { } /** - * @param {import('protomux')} protomux A protomux instance + * Bound to `this` + * + * This will be called whenever a peer is successfully added to the creator + * core, which means that the peer has the project key. The PeerSyncController + * will then handle validation of role records to ensure that the peer is + * actually still part of the project. + * + * @param {{ protomux: import('protomux') }} peer */ - [kSyncReplicate](protomux) { + #handlePeerAdd = (peer) => { + const { protomux } = peer if (this.#peerSyncControllers.has(protomux)) { this.#l.log( - 'Existing sync controller for peer %h', + 'Unexpected existing peer sync controller for peer %h', protomux.stream.remotePublicKey ) return } - const peerSyncController = new PeerSyncController({ protomux, coreManager: this.#coreManager, @@ -87,9 +128,39 @@ export class SyncApi extends TypedEmitter { capabilities: this.#capabilities, logger: this.#l, }) + this.#peerSyncControllers.set(protomux, peerSyncController) + if (this.#dataSyncEnabled.has('local')) { peerSyncController.enableDataSync() } - this.#peerSyncControllers.set(protomux, peerSyncController) + + const peerQueue = this.#pendingDiscoveryKeys.get(protomux) + if (peerQueue) { + for (const discoveryKey of peerQueue) { + peerSyncController.handleDiscoveryKey(discoveryKey) + } + this.#pendingDiscoveryKeys.delete(protomux) + } + } + + /** + * Bound to `this` + * + * Called when a peer is removed from the creator core, e.g. when the + * connection is terminated. + * + * @param {{ protomux: import('protomux') }} peer + */ + #handlePeerRemove = (peer) => { + const { protomux } = peer + if (!this.#peerSyncControllers.has(protomux)) { + this.#l.log( + 'Unexpected no existing peer sync controller for peer %h', + protomux.stream.remotePublicKey + ) + return + } + this.#peerSyncControllers.delete(protomux) + this.#pendingDiscoveryKeys.delete(protomux) } } diff --git a/tests/blob-server.js b/tests/blob-server.js index 7f8b4cc10..11fc89c6c 100644 --- a/tests/blob-server.js +++ b/tests/blob-server.js @@ -204,7 +204,7 @@ test('GET photo returns 404 when trying to get non-replicated blob', async (t) = await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) /** @type {any}*/ - const replicatedCore = cm2.getCoreByDiscoveryKey( + const { core: replicatedCore } = cm2.getCoreByDiscoveryKey( Buffer.from(blobId.driveId, 'hex') ) await replicatedCore.update({ wait: true }) diff --git a/tests/blob-store/blob-store.js b/tests/blob-store/blob-store.js index 7dc5558cb..254e73a48 100644 --- a/tests/blob-store/blob-store.js +++ b/tests/blob-store/blob-store.js @@ -90,7 +90,9 @@ test('get(), initialized but unreplicated drive', async (t) => { await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) /** @type {any} */ - const replicatedCore = cm2.getCoreByDiscoveryKey(Buffer.from(driveId, 'hex')) + const { core: replicatedCore } = cm2.getCoreByDiscoveryKey( + Buffer.from(driveId, 'hex') + ) await replicatedCore.update({ wait: true }) await destroy() t.is(replicatedCore.contiguousLength, 0, 'data is not downloaded') @@ -114,7 +116,9 @@ test('get(), replicated blobIndex, but blobs not replicated', async (t) => { const { destroy } = replicateBlobs(cm1, cm2) await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) /** @type {any} */ - const replicatedCore = cm2.getCoreByDiscoveryKey(Buffer.from(driveId, 'hex')) + const { core: replicatedCore } = cm2.getCoreByDiscoveryKey( + Buffer.from(driveId, 'hex') + ) await replicatedCore.update({ wait: true }) await replicatedCore.download({ end: replicatedCore.length }).done() await destroy() From 7513788884a797516a80f250316cfde0c04d55ac Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 10 Nov 2023 17:02:25 +0900 Subject: [PATCH 32/69] update package-lock --- package-lock.json | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index 550b10705..d917d120a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,8 +14,8 @@ "@fastify/type-provider-typebox": "^3.3.0", "@hyperswarm/secret-stream": "^6.1.2", "@mapeo/crypto": "1.0.0-alpha.10", - "@mapeo/schema": "3.0.0-next.11", - "@mapeo/sqlite-indexer": "1.0.0-alpha.6", + "@mapeo/schema": "3.0.0-next.13", + "@mapeo/sqlite-indexer": "1.0.0-alpha.8", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", "base32.js": "^0.1.0", @@ -824,9 +824,9 @@ } }, "node_modules/@mapeo/schema": { - "version": "3.0.0-next.11", - "resolved": "https://registry.npmjs.org/@mapeo/schema/-/schema-3.0.0-next.11.tgz", - "integrity": "sha512-tZnIzNmXpKNSkqEZQP9rCb91toKga/jrSF9RIUsYeIMamsePHtLuTxF2BEVdT81/P+NLgHt57uXIXEIefU3Usw==", + "version": "3.0.0-next.13", + "resolved": "https://registry.npmjs.org/@mapeo/schema/-/schema-3.0.0-next.13.tgz", + "integrity": "sha512-g5+Lx0uGzq5i2nlrDvuPExkzrQpzx3dhl1G1gfXm72Hw8he2ecGOm5Gu9vW9PsfKtN45AKRy+jd7kfl9+jLhpw==", "dependencies": { "@json-schema-tools/dereferencer": "^1.6.1", "ajv": "^8.12.0", @@ -922,9 +922,9 @@ } }, "node_modules/@mapeo/sqlite-indexer": { - "version": "1.0.0-alpha.6", - "resolved": "https://registry.npmjs.org/@mapeo/sqlite-indexer/-/sqlite-indexer-1.0.0-alpha.6.tgz", - "integrity": "sha512-iLUePxr2kHgsWfFTuJAKjTSZCRuVsIVNbQVyLEkN0pX/2dWzljCxCRvO+9rc1x+bThUas96ZAzCedqeeqC0zRw==", + "version": "1.0.0-alpha.8", + "resolved": "https://registry.npmjs.org/@mapeo/sqlite-indexer/-/sqlite-indexer-1.0.0-alpha.8.tgz", + "integrity": "sha512-qU+I6L4QKp6CkNA5AYu8dqADWaX+usfZq89c+fKOmIRZ+jR9ta3790PzCQbr5VCaYPDfp0OfemO1EjJ7RhHfcQ==", "dependencies": { "@types/better-sqlite3": "^7.6.4", "better-sqlite3": "^8.4.0" From 8c6a081cd0ef6906e1040c2850dc30c235b07f3c Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 10 Nov 2023 17:56:59 +0900 Subject: [PATCH 33/69] chore: Add debug logging --- package-lock.json | 3 +- src/core-manager/index.js | 23 ++++++++++- src/index-writer/index.js | 17 +++++++- src/local-peers.js | 71 +++++++++++++++++++++++++++++--- src/logger.js | 69 +++++++++++++++++++++++++++++++ src/mapeo-manager.js | 28 ++++++++++--- src/mapeo-project.js | 12 +++++- src/sync/peer-sync-controller.js | 31 ++++++++++++-- src/sync/sync-api.js | 17 +++++++- 9 files changed, 250 insertions(+), 21 deletions(-) create mode 100644 src/logger.js diff --git a/package-lock.json b/package-lock.json index fcefb63be..5e30ab7cd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2247,7 +2247,8 @@ }, "node_modules/debug": { "version": "4.3.4", - "license": "MIT", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.4.tgz", + "integrity": "sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==", "dependencies": { "ms": "2.1.2" }, diff --git a/src/core-manager/index.js b/src/core-manager/index.js index e548f2d94..3f085738d 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -8,6 +8,7 @@ import { HaveExtension, ProjectExtension } from '../generated/extensions.js' import { CoreIndex } from './core-index.js' import { ReplicationStateMachine } from './replication-state-machine.js' import * as rle from './bitfield-rle.js' +import { Logger } from '../logger.js' // WARNING: Changing these will break things for existing apps, since namespaces // are used for key derivation @@ -54,6 +55,8 @@ export class CoreManager extends TypedEmitter { #state = 'opened' #ready #haveExtension + #deviceId + #l static get namespaces() { return NAMESPACES @@ -67,6 +70,7 @@ export class CoreManager extends TypedEmitter { * @param {Buffer} [options.projectSecretKey] 32-byte secret key of the project creator core * @param {Partial>} [options.encryptionKeys] Encryption keys for each namespace * @param {import('hypercore').HypercoreStorage} options.storage Folder to store all hypercore data + * @param {Logger} [options.logger] */ constructor({ sqlite, @@ -75,6 +79,7 @@ export class CoreManager extends TypedEmitter { projectSecretKey, encryptionKeys = {}, storage, + logger, }) { super() assert( @@ -85,7 +90,9 @@ export class CoreManager extends TypedEmitter { !projectSecretKey || projectSecretKey.length === 64, 'project owner core secret key must be 64-byte buffer' ) + this.#l = Logger.create('coreManager', logger) const primaryKey = keyManager.getDerivedKey('primaryKey', projectKey) + this.#deviceId = keyManager.getIdentityKeypair().publicKey.toString('hex') this.#projectKey = projectKey this.#encryptionKeys = encryptionKeys @@ -157,7 +164,15 @@ export class CoreManager extends TypedEmitter { this.#ready = Promise.all( [...this.#coreIndex].map(({ core }) => core.ready()) - ).catch(() => {}) + ) + .then(() => { + this.#l.log('ready') + }) + .catch(() => {}) + } + + get deviceId() { + return this.#deviceId } get creatorCore() { @@ -304,6 +319,12 @@ export class CoreManager extends TypedEmitter { this.#addCoreSqlStmt.run({ publicKey: key, namespace }) } + this.#l.log( + 'Added %s %s core %k', + persist ? 'remote' : writer ? 'local' : 'creator', + namespace, + key + ) this.emit('add-core', { core, key, namespace }) return { core, key, namespace } diff --git a/src/index-writer/index.js b/src/index-writer/index.js index b131e2468..a48c5fde1 100644 --- a/src/index-writer/index.js +++ b/src/index-writer/index.js @@ -3,6 +3,7 @@ import SqliteIndexer from '@mapeo/sqlite-indexer' import { getTableConfig } from 'drizzle-orm/sqlite-core' import { getBacklinkTableName } from '../schema/utils.js' import { discoveryKey } from 'hypercore-crypto' +import { Logger } from '../logger.js' /** * @typedef {import('../datatype/index.js').MapeoDocTables} MapeoDocTables @@ -21,6 +22,7 @@ export class IndexWriter { /** @type {Map} */ #indexers = new Map() #mapDoc + #l /** * * @param {object} opts @@ -28,8 +30,10 @@ export class IndexWriter { * @param {TTables[]} opts.tables * @param {(doc: MapeoDocInternal, version: import('@mapeo/schema').VersionIdObject) => MapeoDoc} [opts.mapDoc] optionally transform a document prior to indexing. Can also validate, if an error is thrown then the document will not be indexed * @param {typeof import('@mapeo/sqlite-indexer').defaultGetWinner} [opts.getWinner] custom function to determine the "winner" of two forked documents. Defaults to choosing the document with the most recent `updatedAt` + * @param {Logger} [opts.logger] */ - constructor({ tables, sqlite, mapDoc = (d) => d, getWinner }) { + constructor({ tables, sqlite, mapDoc = (d) => d, getWinner, logger }) { + this.#l = Logger.create('indexWriter', logger) this.#mapDoc = mapDoc for (const table of tables) { const config = getTableConfig(table) @@ -63,6 +67,7 @@ export class IndexWriter { const version = { coreDiscoveryKey: discoveryKey(key), index } var doc = this.#mapDoc(decode(block, version), version) } catch (e) { + this.#l.log('Could not decode entry %d of %h', index, key) // Unknown or invalid entry - silently ignore continue } @@ -80,6 +85,16 @@ export class IndexWriter { continue } indexer.batch(docs) + if (this.#l.enabled) { + for (const doc of docs) { + this.#l.log( + 'Indexed %s %S @ %S', + doc.schemaName, + doc.docId, + doc.versionId + ) + } + } } } } diff --git a/src/local-peers.js b/src/local-peers.js index 32c0d2305..14424e63e 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -10,6 +10,7 @@ import { InviteResponse_Decision, } from './generated/rpc.js' import pDefer from 'p-defer' +import { Logger } from './logger.js' const PROTOCOL_NAME = 'mapeo/rpc' @@ -59,16 +60,23 @@ class Peer { #disconnectedAt = 0 /** @type {Protomux} */ #protomux + #log /** * @param {object} options * @param {Buffer} options.publicKey * @param {ReturnType} options.channel + * @param {Logger} [options.logger] */ - constructor({ publicKey, channel }) { + constructor({ publicKey, channel, logger }) { this.#publicKey = publicKey this.#channel = channel this.#connected = pDefer() + // @ts-ignore + this.#log = (formatter, ...args) => { + const log = Logger.create('peer', logger).log + return log.apply(null, [`[%h] ${formatter}`, publicKey, ...args]) + } } /** @returns {PeerInfoInternal} */ get info() { @@ -108,26 +116,34 @@ class Peer { this.#protomux = protomux /* c8 ignore next 3 */ if (this.#state !== 'connecting') { + this.#log('ERROR: tried to connect but state was %s', this.#state) return // TODO: report error - this should not happen } this.#state = 'connected' this.#connectedAt = Date.now() this.#connected.resolve() + this.#log('connected') } disconnect() { // @ts-ignore - easier to ignore this than handle this for TS - avoids holding a reference to old Protomux instances this.#protomux = undefined /* c8 ignore next */ - if (this.#state === 'disconnected') return + if (this.#state === 'disconnected') { + this.#log('ERROR: tried to disconnect but was already disconnected') + return + } this.#state = 'disconnected' this.#disconnectedAt = Date.now() // Can just resolve this rather than reject, because #assertConnected will throw the error this.#connected.resolve() + let rejectCount = 0 for (const pending of this.pendingInvites.values()) { for (const { reject } of pending) { reject(new PeerDisconnectedError()) + rejectCount++ } } + this.#log('disconnected and rejected %d pending invites', rejectCount) this.pendingInvites.clear() } /** @param {InviteWithKeys} invite */ @@ -136,6 +152,7 @@ class Peer { const buf = Buffer.from(Invite.encode(invite).finish()) const messageType = MESSAGE_TYPES.Invite this.#channel.messages[messageType].send(buf) + this.#log('sent invite for %h', invite.projectKey) } /** @param {InviteResponse} response */ async sendInviteResponse(response) { @@ -143,6 +160,11 @@ class Peer { const buf = Buffer.from(InviteResponse.encode(response).finish()) const messageType = MESSAGE_TYPES.InviteResponse this.#channel.messages[messageType].send(buf) + this.#log( + 'sent response for %h: %s', + response.projectKey, + response.decision + ) } /** @param {DeviceInfo} deviceInfo */ async sendDeviceInfo(deviceInfo) { @@ -150,10 +172,12 @@ class Peer { const buf = Buffer.from(DeviceInfo.encode(deviceInfo).finish()) const messageType = MESSAGE_TYPES.DeviceInfo this.#channel.messages[messageType].send(buf) + this.#log('sent deviceInfo %o', deviceInfo) } /** @param {DeviceInfo} deviceInfo */ receiveDeviceInfo(deviceInfo) { this.#name = deviceInfo.name + this.#log('received deviceInfo %o', deviceInfo) } async #assertConnected() { await this.#connected.promise @@ -179,6 +203,17 @@ export class LocalPeers extends TypedEmitter { #opening = new Set() static InviteResponse = InviteResponse_Decision + #l + + /** + * + * @param {object} [opts] + * @param {Logger} [opts.logger] + */ + constructor({ logger } = {}) { + super() + this.#l = Logger.create('localPeers', logger) + } /** * Invite a peer to a project. Resolves with the response from the invitee: @@ -195,7 +230,6 @@ export class LocalPeers extends TypedEmitter { async invite(peerId, { timeout, ...invite }) { await Promise.all(this.#opening) const peer = this.#peers.get(peerId) - if (!peer) console.log([...this.#peers.keys()]) if (!peer) throw new UnknownPeerError('Unknown peer ' + peerId) /** @type {Promise} */ return new Promise((origResolve, origReject) => { @@ -277,6 +311,11 @@ export class LocalPeers extends TypedEmitter { protomux.pair( { protocol: 'hypercore/alpha' }, /** @param {Buffer} discoveryKey */ async (discoveryKey) => { + this.#l.log( + 'Received dk %h from %h', + discoveryKey, + stream.noiseStream.remotePublicKey + ) this.emit('discovery-key', discoveryKey, stream.rawStream) } ) @@ -288,7 +327,13 @@ export class LocalPeers extends TypedEmitter { // opened, so this helped awaits the open openedNoiseSecretStream(stream).then((stream) => { this.#opening.delete(stream.opened) - if (stream.destroyed) return + if (stream.destroyed) { + this.#l.log( + 'Opened connection to %h but was already destroyed', + stream.remotePublicKey + ) + return + } const { remotePublicKey } = stream // This is written like this because the protomux uses the index within @@ -318,7 +363,11 @@ export class LocalPeers extends TypedEmitter { if (existingPeer && existingPeer.info.status !== 'disconnected') { existingPeer.disconnect() // Should not happen, but in case } - const peer = new Peer({ publicKey: remotePublicKey, channel }) + const peer = new Peer({ + publicKey: remotePublicKey, + channel, + logger: this.#l, + }) this.#peers.set(peerId, peer) // Do not emit peers now - will emit when connected }) @@ -345,7 +394,10 @@ export class LocalPeers extends TypedEmitter { const peerId = publicKey.toString('hex') const peer = this.#peers.get(peerId) /* c8 ignore next */ - if (!peer) return // TODO: report error - this should not happen + if (!peer) { + this.#l.log('ERROR: Could not close peer %h', publicKey) + return // TODO: report error - this should not happen + } // No-op if no change in state /* c8 ignore next */ if (peer.info.status === 'disconnected') return @@ -384,6 +436,7 @@ export class LocalPeers extends TypedEmitter { const invite = Invite.decode(value) assertInviteHasKeys(invite) this.emit('invite', peerId, invite) + this.#l.log('Invite from %h for %h', peerPublicKey, invite.projectKey) break } case 'InviteResponse': { @@ -397,6 +450,12 @@ export class LocalPeers extends TypedEmitter { for (const deferredPromise of pending) { deferredPromise.resolve(response.decision) } + this.#l.log( + 'Invite response from %h for %h: %s', + peerPublicKey, + response.projectKey, + response.decision + ) peer.pendingInvites.set(projectId, []) break } diff --git a/src/logger.js b/src/logger.js new file mode 100644 index 000000000..356e6cc35 --- /dev/null +++ b/src/logger.js @@ -0,0 +1,69 @@ +import createDebug from 'debug' +import { discoveryKey } from 'hypercore-crypto' + +const TRIM = 7 + +createDebug.formatters.h = (v) => { + if (!Buffer.isBuffer(v)) return '[undefined]' + return v.toString('hex').slice(0, TRIM) +} + +createDebug.formatters.S = (v) => { + if (typeof v !== 'string') return '[undefined]' + return v.slice(0, 7) +} + +createDebug.formatters.k = (v) => { + if (!Buffer.isBuffer(v)) return '[undefined]' + return discoveryKey(v).toString('hex').slice(0, TRIM) +} + +const counts = new Map() + +export class Logger { + #baseLogger + #log + + /** + * @param {string} ns + * @param {Logger} [logger] + */ + static create(ns, logger) { + if (logger) return logger.extend(ns) + const i = (counts.get(ns) || 0) + 1 + const deviceId = String(i).padStart(TRIM, '0') + return new Logger({ deviceId, ns }) + } + + /** + * @param {object} opts + * @param {string} opts.deviceId + * @param {createDebug.Debugger} [opts.baseLogger] + * @param {string} [opts.ns] + */ + constructor({ deviceId, baseLogger, ns }) { + this.deviceId = deviceId + this.#baseLogger = baseLogger || createDebug('mapeo' + (ns ? `:${ns}` : '')) + this.#log = this.#baseLogger.extend(this.deviceId.slice(0, TRIM)) + } + get enabled() { + return this.#log.enabled + } + + /** + * @param {Parameters} args + */ + log = (...args) => { + this.#log.apply(this, args) + } + /** + * + * @param {string} ns + */ + extend(ns) { + return new Logger({ + deviceId: this.deviceId, + baseLogger: this.#baseLogger.extend(ns), + }) + } +} diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index a21adb6e2..dcac54bed 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -30,6 +30,7 @@ import { LocalPeers } from './local-peers.js' import { InviteApi } from './invite-api.js' import { MediaServer } from './media-server.js' import { LocalDiscovery } from './discovery/local-discovery.js' +import { Logger } from './logger.js' /** @typedef {import("@mapeo/schema").ProjectSettingsValue} ProjectValue */ @@ -71,6 +72,7 @@ export class MapeoManager extends TypedEmitter { #invite #mediaServer #localDiscovery + #l /** * @param {Object} opts @@ -81,7 +83,9 @@ export class MapeoManager extends TypedEmitter { */ constructor({ rootKey, dbFolder, coreStorage, mediaServerOpts }) { super() - + this.#keyManager = new KeyManager(rootKey) + this.#deviceId = getDeviceId(this.#keyManager) + this.#l = new Logger({ deviceId: this.#deviceId }) this.#dbFolder = dbFolder const sqlite = new Database( dbFolder === ':memory:' @@ -93,16 +97,15 @@ export class MapeoManager extends TypedEmitter { migrationsFolder: new URL('../drizzle/client', import.meta.url).pathname, }) - this.#localPeers = new LocalPeers() + this.#localPeers = new LocalPeers({ logger: this.#l }) this.#localPeers.on('peers', (peers) => { this.emit('local-peers', omitPeerProtomux(peers)) }) - this.#keyManager = new KeyManager(rootKey) - this.#deviceId = getDeviceId(this.#keyManager) this.#projectSettingsIndexWriter = new IndexWriter({ tables: [projectSettingsTable], sqlite, + logger: this.#l, }) this.#activeProjects = new Map() @@ -150,6 +153,10 @@ export class MapeoManager extends TypedEmitter { return this.#localPeers } + get deviceId() { + return this.#deviceId + } + /** * Replicate Mapeo to a `@hyperswarm/secret-stream`. This replication connects * the Mapeo RPC channel and allows invites. All active projects will sync @@ -169,7 +176,11 @@ export class MapeoManager extends TypedEmitter { }) .catch((e) => { // Ignore error but log - console.error('Failed to send device info to peer', e) + this.#l.log( + 'Failed to send device info to peer %h', + noiseStream.remotePublicKey, + e + ) }) return replicationStream } @@ -285,6 +296,12 @@ export class MapeoManager extends TypedEmitter { // TODO: Close the project instance instead of keeping it around this.#activeProjects.set(projectPublicId, project) + this.#l.log( + 'created project %h, public id: %S', + projectKeypair.publicKey, + projectPublicId + ) + // 7. Return project public id return projectPublicId } @@ -338,6 +355,7 @@ export class MapeoManager extends TypedEmitter { sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, localPeers: this.#localPeers, + logger: this.#l, getMediaBaseUrl: this.#mediaServer.getMediaAddress.bind( this.#mediaServer ), diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 11ede268c..ba06a1e87 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -39,6 +39,7 @@ import { MemberApi } from './member-api.js' import { IconApi } from './icon-api.js' import { SyncApi, kSyncReplicate } from './sync/sync-api.js' import Hypercore from 'hypercore' +import { Logger } from './logger.js' /** @typedef {Omit} EditableProjectSettings */ @@ -64,6 +65,7 @@ export class MapeoProject { #projectPublicId #iconApi #syncApi + #l /** * @param {Object} opts @@ -77,6 +79,7 @@ export class MapeoProject { * @param {import('./types.js').CoreStorage} opts.coreStorage Folder to store all hypercore data * @param {(mediaType: 'blobs' | 'icons') => Promise} opts.getMediaBaseUrl * @param {import('./local-peers.js').LocalPeers} opts.localPeers + * @param {Logger} [opts.logger] * */ constructor({ @@ -90,7 +93,9 @@ export class MapeoProject { encryptionKeys, getMediaBaseUrl, localPeers, + logger, }) { + this.#l = Logger.create('project', logger) this.#deviceId = getDeviceId(keyManager) this.#projectId = projectKeyToId(projectKey) this.#projectPublicId = projectKeyToPublicId(projectKey) @@ -121,6 +126,7 @@ export class MapeoProject { keyManager, storage: coreManagerStorage, sqlite, + logger: this.#l, }) const indexWriter = new IndexWriter({ @@ -144,6 +150,7 @@ export class MapeoProject { return doc } }, + logger: this.#l, }) this.#dataStores = { auth: new DataStore({ @@ -260,6 +267,7 @@ export class MapeoProject { this.#syncApi = new SyncApi({ coreManager: this.#coreManager, capabilities: this.#capabilities, + logger: this.#l, }) ///////// 4. Wire up sync @@ -305,6 +313,7 @@ export class MapeoProject { .then(deferred.resolve) .catch(deferred.reject) }) + this.#l.log('Created project instance %h', projectKey) } /** @@ -428,7 +437,8 @@ export class MapeoProject { return extractEditableProjectSettings( await this.#dataTypes.projectSettings.getByDocId(this.#projectId) ) - } catch { + } catch (e) { + this.#l.log('No project settings') return /** @type {EditableProjectSettings} */ ({}) } } diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index 01cf76ff7..306525b8e 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -1,5 +1,6 @@ import mapObject from 'map-obj' import { NAMESPACES } from '../core-manager/index.js' +import { Logger } from '../logger.js' /** * @typedef {import('../core-manager/index.js').Namespace} Namespace @@ -29,6 +30,7 @@ export class PeerSyncController { #downloadingRanges = new Map() /** @type {SyncStatus} */ #prevSyncStatus = createNamespaceMap('unknown') + #log /** * @param {object} opts @@ -36,8 +38,18 @@ export class PeerSyncController { * @param {import("../core-manager/index.js").CoreManager} opts.coreManager * @param {import("./sync-state.js").SyncState} opts.syncState * @param {import("../capabilities.js").Capabilities} opts.capabilities + * @param {Logger} [opts.logger] */ - constructor({ protomux, coreManager, syncState, capabilities }) { + constructor({ protomux, coreManager, syncState, capabilities, logger }) { + // @ts-ignore + this.#log = (formatter, ...args) => { + const log = Logger.create('peer', logger).log + return log.apply(null, [ + `[%h] ${formatter}`, + protomux.stream.remotePublicKey, + ...args, + ]) + } this.#coreManager = coreManager this.#protomux = protomux this.#capabilities = capabilities @@ -52,6 +64,14 @@ export class PeerSyncController { this.#updateEnabledNamespaces() } + get peerKey() { + return this.#protomux.stream.remotePublicKey + } + + get peerId() { + return this.peerKey?.toString('hex') + } + /** * Enable syncing of data (in the data and blob namespaces) */ @@ -99,6 +119,7 @@ export class PeerSyncController { const localState = mapObject(state, (ns, nsState) => { return [ns, nsState.localState] }) + this.#log('state %O', state) // Map of which namespaces have received new data since last sync change const didUpdate = mapObject(state, (ns) => { @@ -121,13 +142,12 @@ export class PeerSyncController { const cap = await this.#capabilities.getCapabilities(peerId) this.#syncCapability = cap.sync } catch (e) { + this.#log('Error reading capability', e) // Any error, consider sync blocked this.#syncCapability = createNamespaceMap('blocked') } } - // console.log(peerId.slice(0, 7), this.#syncCapability) - // console.log(peerId.slice(0, 7), didUpdate) - // console.dir(state, { depth: null, colors: true }) + this.#log('capability %o', this.#syncCapability) // If any namespace has new data, update what is enabled if (Object.values(didUpdate).indexOf(true) > -1) { @@ -190,6 +210,7 @@ export class PeerSyncController { (peer) => peer.protomux === this.#protomux ) if (!peerToUnreplicate) return + this.#log('unreplicating core %k', core.key) peerToUnreplicate.channel.close() this.#replicatingCores.delete(core) } @@ -222,6 +243,7 @@ export class PeerSyncController { this.#downloadCore(core) } this.#enabledNamespaces.add(namespace) + this.#log('enabled namespace %s', namespace) } /** @@ -233,6 +255,7 @@ export class PeerSyncController { this.#undownloadCore(core) } this.#enabledNamespaces.delete(namespace) + this.#log('disabled namespace %s', namespace) } } diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index 9508adc13..5eaf685db 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -1,6 +1,7 @@ import { TypedEmitter } from 'tiny-typed-emitter' import { SyncState } from './sync-state.js' import { PeerSyncController } from './peer-sync-controller.js' +import { Logger } from '../logger.js' export const kSyncReplicate = Symbol('replicate sync') @@ -20,6 +21,7 @@ export class SyncApi extends TypedEmitter { #peerSyncControllers = new Map() /** @type {Set<'local' | 'remote'>} */ #dataSyncEnabled = new Set() + #l /** * @@ -27,9 +29,11 @@ export class SyncApi extends TypedEmitter { * @param {import('../core-manager/index.js').CoreManager} opts.coreManager * @param {import("../capabilities.js").Capabilities} opts.capabilities * @param {number} [opts.throttleMs] + * @param {Logger} [opts.logger] */ - constructor({ coreManager, throttleMs = 200, capabilities }) { + constructor({ coreManager, throttleMs = 200, capabilities, logger }) { super() + this.#l = Logger.create('syncApi', logger) this.#coreManager = coreManager this.#capabilities = capabilities this.syncState = new SyncState({ coreManager, throttleMs }) @@ -46,6 +50,7 @@ export class SyncApi extends TypedEmitter { start() { if (this.#dataSyncEnabled.has('local')) return this.#dataSyncEnabled.add('local') + this.#l.log('Starting data sync') for (const peerSyncController of this.#peerSyncControllers.values()) { peerSyncController.enableDataSync() } @@ -57,6 +62,7 @@ export class SyncApi extends TypedEmitter { stop() { if (!this.#dataSyncEnabled.has('local')) return this.#dataSyncEnabled.delete('local') + this.#l.log('Stopping data sync') for (const peerSyncController of this.#peerSyncControllers.values()) { peerSyncController.disableDataSync() } @@ -66,13 +72,20 @@ export class SyncApi extends TypedEmitter { * @param {import('protomux')} protomux A protomux instance */ [kSyncReplicate](protomux) { - if (this.#peerSyncControllers.has(protomux)) return + if (this.#peerSyncControllers.has(protomux)) { + this.#l.log( + 'Unexpected existing peer sync controller for peer %h', + protomux.stream.remotePublicKey + ) + return + } const peerSyncController = new PeerSyncController({ protomux, coreManager: this.#coreManager, syncState: this.syncState, capabilities: this.#capabilities, + logger: this.#l, }) if (this.#dataSyncEnabled.has('local')) { peerSyncController.enableDataSync() From 8a637466716aa1a01eb1f506f95bc11610addc4f Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 10 Nov 2023 20:46:44 +0900 Subject: [PATCH 34/69] Add new logger to discovery + dnssd --- src/discovery/dns-sd.js | 12 +++++++----- src/discovery/local-discovery.js | 24 ++++++++++-------------- src/local-peers.js | 2 +- src/mapeo-manager.js | 3 ++- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/src/discovery/dns-sd.js b/src/discovery/dns-sd.js index 7f68531c2..677006db3 100644 --- a/src/discovery/dns-sd.js +++ b/src/discovery/dns-sd.js @@ -1,10 +1,9 @@ import { TypedEmitter } from 'tiny-typed-emitter' import { Bonjour } from 'bonjour-service' -// @ts-ignore -import debug from 'debug' import pTimeout from 'p-timeout' import { randomBytes } from 'node:crypto' import { once } from 'node:events' +import { Logger } from '../logger.js' const SERVICE_NAME = 'mapeo' @@ -48,7 +47,7 @@ export class DnsSd extends TypedEmitter { } /* c8 ignore stop */ const { name, port } = service - this.#log(`service up`, [name, address, port]) + this.#log('serviceUp', name.slice(0, 7), address, port) this.emit('up', { port, name, address }) } /** @param {import('bonjour-service').Service} service */ @@ -74,18 +73,21 @@ export class DnsSd extends TypedEmitter { /** @type {Promise | null} */ #advertisingStopping = null #log + #l /** * * @param {object} [opts] * @param {string} [opts.name] * @param {boolean} [opts.disableIpv6] + * @param {Logger} [opts.logger] */ - constructor({ name, disableIpv6 = true } = {}) { + constructor({ name, disableIpv6 = true, logger } = {}) { super() + this.#l = Logger.create('dnssd', logger) this.#name = name || randomBytes(8).toString('hex') this.#disableIpv6 = disableIpv6 - this.#log = debug('mapeo:dnssd:' + this.#name) + this.#log = this.#l.log.bind(this.#l) } get name() { diff --git a/src/discovery/local-discovery.js b/src/discovery/local-discovery.js index 6c1ade7bb..6d3d59eed 100644 --- a/src/discovery/local-discovery.js +++ b/src/discovery/local-discovery.js @@ -3,11 +3,11 @@ import net from 'node:net' import NoiseSecretStream from '@hyperswarm/secret-stream' import { once } from 'node:events' import { DnsSd } from './dns-sd.js' -import debug from 'debug' import { isPrivate } from 'bogon' import StartStopStateMachine from 'start-stop-state-machine' import pTimeout from 'p-timeout' import { keyToPublicId } from '@mapeo/crypto' +import { Logger } from '../logger.js' /** @typedef {{ publicKey: Buffer, secretKey: Buffer }} Keypair */ /** @typedef {import('../utils.js').OpenedNoiseStream} OpenedNoiseStream */ @@ -32,21 +32,25 @@ export class LocalDiscovery extends TypedEmitter { #log /** @type {(e: Error) => void} */ #handleSocketError + #l /** * @param {Object} opts * @param {Keypair} opts.identityKeypair * @param {DnsSd} [opts.dnssd] Optional DnsSd instance, used for testing + * @param {Logger} [opts.logger] */ - constructor({ identityKeypair, dnssd }) { + constructor({ identityKeypair, dnssd, logger }) { super() + this.#l = Logger.create('mdns', logger) + this.#log = this.#l.log.bind(this.#l) this.#dnssd = dnssd || new DnsSd({ name: keyToPublicId(identityKeypair.publicKey), + logger: this.#l, }) this.#dnssd.on('up', this.#handleServiceUp.bind(this)) - this.#log = debug('mapeo:mdns:' + keyShortname(identityKeypair.publicKey)) this.#sm = new StartStopStateMachine({ start: this.#start.bind(this), stop: this.#stop.bind(this), @@ -195,7 +199,8 @@ export class LocalDiscovery extends TypedEmitter { this.#log( `${isInitiator ? 'outgoing' : 'incoming'} secretSteam connection ${ isInitiator ? 'to' : 'from' - } ${keyShortname(remotePublicKey)}` + } %h`, + remotePublicKey ) const existing = this.#noiseConnections.get(remoteId) @@ -231,7 +236,7 @@ export class LocalDiscovery extends TypedEmitter { this.#noiseConnections.set(remoteId, conn) conn.on('close', () => { - this.#log(`closed connection with ${keyShortname(remotePublicKey)}`) + this.#log('closed connection with %h', remotePublicKey) this.#noiseConnections.delete(remoteId) }) @@ -295,13 +300,4 @@ function getAddress(server) { return addr } -/** - * - * @param {Buffer} key - * @returns - */ -function keyShortname(key) { - return keyToPublicId(key).slice(0, 7) -} - function noop() {} diff --git a/src/local-peers.js b/src/local-peers.js index 14424e63e..1a38360d0 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -312,7 +312,7 @@ export class LocalPeers extends TypedEmitter { { protocol: 'hypercore/alpha' }, /** @param {Buffer} discoveryKey */ async (discoveryKey) => { this.#l.log( - 'Received dk %h from %h', + 'Received discovery key %h from %h', discoveryKey, stream.noiseStream.remotePublicKey ) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index c6e3312da..5e37ee89a 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -87,7 +87,7 @@ export class MapeoManager extends TypedEmitter { super() this.#keyManager = new KeyManager(rootKey) this.#deviceId = getDeviceId(this.#keyManager) - this.#l = new Logger({ deviceId: this.#deviceId }) + this.#l = new Logger({ deviceId: this.#deviceId, ns: 'manager' }) this.#dbFolder = dbFolder const sqlite = new Database( dbFolder === ':memory:' @@ -149,6 +149,7 @@ export class MapeoManager extends TypedEmitter { this.#localDiscovery = new LocalDiscovery({ identityKeypair: this.#keyManager.getIdentityKeypair(), + logger: this.#l, }) this.#localDiscovery.on('connection', this.#replicate.bind(this)) } From ed69a78ae43418eb8c832db8ddb79b702ad144d9 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 10 Nov 2023 21:53:16 +0900 Subject: [PATCH 35/69] Get invite test working --- src/local-peers.js | 8 ++++- src/mapeo-manager.js | 9 ++++++ test-e2e/manager-invite.js | 66 ++++++++++++++++++++++++++++---------- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index 1a38360d0..e68347385 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -316,10 +316,16 @@ export class LocalPeers extends TypedEmitter { discoveryKey, stream.noiseStream.remotePublicKey ) - this.emit('discovery-key', discoveryKey, stream.rawStream) + this.emit('discovery-key', discoveryKey, stream) } ) + protomux.pair({ protocol: PROTOCOL_NAME }, async () => { + // Seem to need this because of the async tick below waiting for the noise + // stream to open + await stream.opened + }) + // No need to connect error handler to stream because Protomux does this, // and errors are eventually handled by #closePeer diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 5e37ee89a..ca94ad49e 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -636,6 +636,15 @@ export class MapeoManager extends TypedEmitter { await this.#mediaServer.stop() } + async startLocalPeerDiscovery() { + return this.#localDiscovery.start() + } + + /** @type {LocalDiscovery['stop']} */ + async stopLocalPeerDiscovery(opts) { + return this.#localDiscovery.stop(opts) + } + /** * @returns {Promise} */ diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index a6dcb8ac3..a72ad36f6 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -2,27 +2,22 @@ import { test } from 'brittle' import RAM from 'random-access-memory' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' -import { MapeoManager, kManagerReplicate } from '../src/mapeo-manager.js' +import { MapeoManager, kManagerReplicate, kRPC } from '../src/mapeo-manager.js' import { once } from 'node:events' import sodium from 'sodium-universal' test('member invite accepted', async (t) => { - const creator = createManager('creator') - await creator.setDeviceInfo({ name: 'Creator' }) + const [creator, joiner] = await createManagers(2) + await connectPeers([creator, joiner]) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) - const joiner = createManager('joiner1') - await joiner.setDeviceInfo({ name: 'Joiner' }) - await t.exception( async () => joiner.getProject(createdProjectId), 'joiner cannot get project instance before being invited and added to project' ) - const destroy = replicate(creator, joiner) - const responsePromise = creatorProject.$member.invite(joiner.deviceId, { roleId: MEMBER_ROLE_ID, }) @@ -61,26 +56,21 @@ test('member invite accepted', async (t) => { 'Project members match' ) - await destroy() + await disconnectPeers([creator, joiner]) }) test('member invite rejected', async (t) => { - const creator = createManager('creator') - await creator.setDeviceInfo({ name: 'Creator' }) + const [creator, joiner] = await createManagers(2) + await connectPeers([creator, joiner]) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) - const joiner = createManager('joiner1') - await joiner.setDeviceInfo({ name: 'Joiner' }) - await t.exception( async () => joiner.getProject(createdProjectId), 'joiner cannot get project instance before being invited and added to project' ) - const destroy = replicate(creator, joiner) - const responsePromise = creatorProject.$member.invite(joiner.deviceId, { roleId: MEMBER_ROLE_ID, }) @@ -114,7 +104,7 @@ test('member invite rejected', async (t) => { 'Only 1 member in project still' ) - await destroy() + await disconnectPeers([creator, joiner]) }) /** @@ -148,6 +138,48 @@ export function replicate(mm1, mm2) { } } +/** + * @param {MapeoManager[]} managers + */ +async function disconnectPeers(managers) { + return Promise.all( + managers.map(async (manager) => { + return manager.stopLocalPeerDiscovery({ force: true }) + }) + ) +} + +/** + * @param {MapeoManager[]} managers + */ +async function connectPeers(managers) { + for (const manager of managers) { + manager.startLocalPeerDiscovery() + } + return new Promise((res) => { + managers[0][kRPC].on('peers', function onPeers(peers) { + if (peers.length !== managers.length - 1) return + if (!peers.every((peerInfo) => peerInfo.status === 'connected')) return + managers[0][kRPC].off('peers', onPeers) + res(null) + }) + }) +} + +/** @param {number} count */ +async function createManagers(count) { + return Promise.all( + Array(count) + .fill(null) + .map(async (_, i) => { + const name = 'device' + i + const manager = createManager(name) + await manager.setDeviceInfo({ name }) + return manager + }) + ) +} + /** @param {string} [seed] */ function createManager(seed) { return new MapeoManager({ From 7300da3d062aeae5b71347fc8d0d0caf9ad79c5a Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 11 Nov 2023 20:59:54 +0900 Subject: [PATCH 36/69] fix manager logger --- src/mapeo-manager.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index ca94ad49e..0243cf9e8 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -74,6 +74,7 @@ export class MapeoManager extends TypedEmitter { #invite #mediaServer #localDiscovery + #loggerBase #l /** @@ -87,7 +88,8 @@ export class MapeoManager extends TypedEmitter { super() this.#keyManager = new KeyManager(rootKey) this.#deviceId = getDeviceId(this.#keyManager) - this.#l = new Logger({ deviceId: this.#deviceId, ns: 'manager' }) + const logger = (this.#loggerBase = new Logger({ deviceId: this.#deviceId })) + this.#l = Logger.create('manager', logger) this.#dbFolder = dbFolder const sqlite = new Database( dbFolder === ':memory:' @@ -99,7 +101,7 @@ export class MapeoManager extends TypedEmitter { migrationsFolder: new URL('../drizzle/client', import.meta.url).pathname, }) - this.#localPeers = new LocalPeers({ logger: this.#l }) + this.#localPeers = new LocalPeers({ logger }) this.#localPeers.on('peers', (peers) => { this.emit('local-peers', omitPeerProtomux(peers)) }) @@ -112,7 +114,7 @@ export class MapeoManager extends TypedEmitter { this.#projectSettingsIndexWriter = new IndexWriter({ tables: [projectSettingsTable], sqlite, - logger: this.#l, + logger, }) this.#activeProjects = new Map() @@ -149,7 +151,7 @@ export class MapeoManager extends TypedEmitter { this.#localDiscovery = new LocalDiscovery({ identityKeypair: this.#keyManager.getIdentityKeypair(), - logger: this.#l, + logger, }) this.#localDiscovery.on('connection', this.#replicate.bind(this)) } @@ -373,7 +375,7 @@ export class MapeoManager extends TypedEmitter { sharedDb: this.#db, sharedIndexWriter: this.#projectSettingsIndexWriter, localPeers: this.#localPeers, - logger: this.#l, + logger: this.#loggerBase, getMediaBaseUrl: this.#mediaServer.getMediaAddress.bind( this.#mediaServer ), @@ -604,6 +606,7 @@ export class MapeoManager extends TypedEmitter { await project[kSetOwnDeviceInfo](deviceInfo) }) ) + this.#l.log('set device info %o', deviceInfo) } /** From b2eda924fc080f22ff5d222859d6ffa99c66f2cd Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Sat, 11 Nov 2023 21:00:13 +0900 Subject: [PATCH 37/69] cleanup invite test (and make it fail :( --- test-e2e/manager-invite.js | 110 ++++--------------------------------- test-e2e/utils-new.js | 99 +++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 100 deletions(-) create mode 100644 test-e2e/utils-new.js diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index a72ad36f6..7cfea1ab8 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -1,14 +1,18 @@ import { test } from 'brittle' -import RAM from 'random-access-memory' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' -import { MapeoManager, kManagerReplicate, kRPC } from '../src/mapeo-manager.js' import { once } from 'node:events' -import sodium from 'sodium-universal' +import { + connectPeers, + createManagers, + disconnectPeers, + waitForPeers, +} from './utils-new.js' test('member invite accepted', async (t) => { const [creator, joiner] = await createManagers(2) - await connectPeers([creator, joiner]) + connectPeers([creator, joiner]) + await waitForPeers([creator, joiner]) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) @@ -61,7 +65,8 @@ test('member invite accepted', async (t) => { test('member invite rejected', async (t) => { const [creator, joiner] = await createManagers(2) - await connectPeers([creator, joiner]) + connectPeers([creator, joiner]) + await waitForPeers([creator, joiner]) const createdProjectId = await creator.createProject({ name: 'Mapeo' }) const creatorProject = await creator.getProject(createdProjectId) @@ -106,98 +111,3 @@ test('member invite rejected', async (t) => { await disconnectPeers([creator, joiner]) }) - -/** - * @param {MapeoManager} mm1 - * @param {MapeoManager} mm2 - */ -export function replicate(mm1, mm2) { - const r1 = mm1[kManagerReplicate](true) - const r2 = mm2[kManagerReplicate](false) - - r1.pipe(r2).pipe(r1) - - /** @param {Error} [e] */ - return async function destroy(e) { - return Promise.all([ - /** @type {Promise} */ - ( - new Promise((res) => { - r1.on('close', res) - r1.destroy(e) - }) - ), - /** @type {Promise} */ - ( - new Promise((res) => { - r2.on('close', res) - r2.destroy(e) - }) - ), - ]) - } -} - -/** - * @param {MapeoManager[]} managers - */ -async function disconnectPeers(managers) { - return Promise.all( - managers.map(async (manager) => { - return manager.stopLocalPeerDiscovery({ force: true }) - }) - ) -} - -/** - * @param {MapeoManager[]} managers - */ -async function connectPeers(managers) { - for (const manager of managers) { - manager.startLocalPeerDiscovery() - } - return new Promise((res) => { - managers[0][kRPC].on('peers', function onPeers(peers) { - if (peers.length !== managers.length - 1) return - if (!peers.every((peerInfo) => peerInfo.status === 'connected')) return - managers[0][kRPC].off('peers', onPeers) - res(null) - }) - }) -} - -/** @param {number} count */ -async function createManagers(count) { - return Promise.all( - Array(count) - .fill(null) - .map(async (_, i) => { - const name = 'device' + i - const manager = createManager(name) - await manager.setDeviceInfo({ name }) - return manager - }) - ) -} - -/** @param {string} [seed] */ -function createManager(seed) { - return new MapeoManager({ - rootKey: getRootKey(seed), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) -} - -/** @param {string} [seed] */ -function getRootKey(seed) { - const key = Buffer.allocUnsafe(16) - if (!seed) { - sodium.randombytes_buf(key) - } else { - const seedBuf = Buffer.alloc(32) - sodium.crypto_generichash(seedBuf, Buffer.from(seed)) - sodium.randombytes_buf_deterministic(key, seedBuf) - } - return key -} diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js new file mode 100644 index 000000000..a5d7e65bf --- /dev/null +++ b/test-e2e/utils-new.js @@ -0,0 +1,99 @@ +// @ts-check +import sodium from 'sodium-universal' +import RAM from 'random-access-memory' + +import { MapeoManager } from '../src/index.js' + +/** + * @param {MapeoManager[]} managers + */ +export async function disconnectPeers(managers) { + return Promise.all( + managers.map(async (manager) => { + return manager.stopLocalPeerDiscovery({ force: true }) + }) + ) +} + +/** + * @param {MapeoManager[]} managers + */ +export function connectPeers(managers, { discovery = true } = {}) { + if (discovery) { + for (const manager of managers) { + manager.startLocalPeerDiscovery() + } + } else { + // TODO: replicate all managers, for faster tests (discovery can take extra time) + } +} + +/** + * Waits for all manager instances to be connected to each other + * + * @param {MapeoManager[]} managers + */ +export async function waitForPeers(managers) { + const peerCounts = Array(managers.length).fill(0) + const expectedCount = managers.length - 1 + return new Promise((res) => { + for (const [idx, manager] of managers.entries()) { + manager.on('local-peers', function onPeers(peers) { + const connectedPeerCount = peers.filter( + ({ status }) => status === 'connected' + ).length + peerCounts[idx] = connectedPeerCount + if (connectedPeerCount === expectedCount) { + manager.off('local-peers', onPeers) + } + if (peerCounts.every((v) => v === expectedCount)) { + res(null) + } + }) + } + }) +} + +/** + * Create `count` manager instances. Each instance has a deterministic identity + * keypair so device IDs should be consistent between tests. + * + * @template {number} T + * @param {T} count + * @returns {Promise>} + */ +export async function createManagers(count) { + // @ts-ignore + return Promise.all( + Array(count) + .fill(null) + .map(async (_, i) => { + const name = 'device' + i + const manager = createManager(name) + await manager.setDeviceInfo({ name }) + return manager + }) + ) +} + +/** @param {string} [seed] */ +export function createManager(seed) { + return new MapeoManager({ + rootKey: getRootKey(seed), + dbFolder: ':memory:', + coreStorage: () => new RAM(), + }) +} + +/** @param {string} [seed] */ +function getRootKey(seed) { + const key = Buffer.allocUnsafe(16) + if (!seed) { + sodium.randombytes_buf(key) + } else { + const seedBuf = Buffer.alloc(32) + sodium.crypto_generichash(seedBuf, Buffer.from(seed)) + sodium.randombytes_buf_deterministic(key, seedBuf) + } + return key +} From 2224f2ba53e6ca1ee2fe2da786c2a3833a3b8d27 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 13:49:09 +0900 Subject: [PATCH 38/69] fix: handle duplicate connections to LocalPeers --- src/local-peers.js | 440 +++++++++++++++++++++++++++++++-------------- 1 file changed, 304 insertions(+), 136 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index 1a38360d0..ba20da8d9 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -1,7 +1,7 @@ // @ts-check import { TypedEmitter } from 'tiny-typed-emitter' import Protomux from 'protomux' -import { openedNoiseSecretStream, keyToId } from './utils.js' +import { keyToId } from './utils.js' import cenc from 'compact-encoding' import { DeviceInfo, @@ -11,8 +11,14 @@ import { } from './generated/rpc.js' import pDefer from 'p-defer' import { Logger } from './logger.js' +import pTimeout, { TimeoutError } from 'p-timeout' +// Unique identifier for the mapeo rpc protocol const PROTOCOL_NAME = 'mapeo/rpc' +// Timeout in milliseconds to wait for a peer to connect when trying to send a message +const SEND_TIMEOUT = 1000 +// Timeout in milliseconds to wait for peer deduplication +const DEDUPE_TIMEOUT = 1000 // Protomux message types depend on the order that messages are added to a // channel (this needs to remain consistent). To avoid breaking changes, the @@ -49,7 +55,7 @@ const MESSAGES_MAX_ID = Math.max.apply(null, [...Object.values(MESSAGE_TYPES)]) class Peer { /** @type {PeerState} */ #state = 'connecting' - #publicKey + #deviceId #channel #connected /** @type {Map>>} */ @@ -58,40 +64,42 @@ class Peer { #name #connectedAt = 0 #disconnectedAt = 0 - /** @type {Protomux} */ #protomux #log /** * @param {object} options - * @param {Buffer} options.publicKey + * @param {string} options.peerId * @param {ReturnType} options.channel + * @param {Protomux} options.protomux * @param {Logger} [options.logger] */ - constructor({ publicKey, channel, logger }) { - this.#publicKey = publicKey + constructor({ peerId, channel, protomux, logger }) { + this.#deviceId = peerId this.#channel = channel + this.#protomux = protomux this.#connected = pDefer() + // Avoid unhandled rejections + this.#connected.promise.catch(noop) // @ts-ignore this.#log = (formatter, ...args) => { const log = Logger.create('peer', logger).log - return log.apply(null, [`[%h] ${formatter}`, publicKey, ...args]) + return log.apply(null, [`[%S] ${formatter}`, peerId, ...args]) } } /** @returns {PeerInfoInternal} */ get info() { - const deviceId = keyToId(this.#publicKey) switch (this.#state) { case 'connecting': return { status: this.#state, - deviceId, + deviceId: this.#deviceId, name: this.#name, } case 'connected': return { status: this.#state, - deviceId, + deviceId: this.#deviceId, name: this.#name, connectedAt: this.#connectedAt, protomux: this.#protomux, @@ -99,7 +107,7 @@ class Peer { case 'disconnected': return { status: this.#state, - deviceId, + deviceId: this.#deviceId, name: this.#name, disconnectedAt: this.#disconnectedAt, } @@ -111,9 +119,18 @@ class Peer { } } } - /** @param {Protomux} protomux */ - connect(protomux) { - this.#protomux = protomux + /** + * A promise that resolves when the peer connects, or rejects if it + * failes to connect + */ + get connected() { + return this.#connected.promise + } + get protomux() { + return this.#protomux + } + + connect() { /* c8 ignore next 3 */ if (this.#state !== 'connecting') { this.#log('ERROR: tried to connect but state was %s', this.#state) @@ -134,8 +151,8 @@ class Peer { } this.#state = 'disconnected' this.#disconnectedAt = Date.now() - // Can just resolve this rather than reject, because #assertConnected will throw the error - this.#connected.resolve() + // This promise should have already resolved, but if the peer never connected then we reject here + this.#connected.reject(new PeerFailedConnectionError()) let rejectCount = 0 for (const pending of this.pendingInvites.values()) { for (const { reject } of pending) { @@ -147,16 +164,16 @@ class Peer { this.pendingInvites.clear() } /** @param {InviteWithKeys} invite */ - async sendInvite(invite) { - await this.#assertConnected() + sendInvite(invite) { + this.#assertConnected() const buf = Buffer.from(Invite.encode(invite).finish()) const messageType = MESSAGE_TYPES.Invite this.#channel.messages[messageType].send(buf) this.#log('sent invite for %h', invite.projectKey) } /** @param {InviteResponse} response */ - async sendInviteResponse(response) { - await this.#assertConnected() + sendInviteResponse(response) { + this.#assertConnected() const buf = Buffer.from(InviteResponse.encode(response).finish()) const messageType = MESSAGE_TYPES.InviteResponse this.#channel.messages[messageType].send(buf) @@ -167,8 +184,7 @@ class Peer { ) } /** @param {DeviceInfo} deviceInfo */ - async sendDeviceInfo(deviceInfo) { - await this.#assertConnected() + sendDeviceInfo(deviceInfo) { const buf = Buffer.from(DeviceInfo.encode(deviceInfo).finish()) const messageType = MESSAGE_TYPES.DeviceInfo this.#channel.messages[messageType].send(buf) @@ -179,8 +195,7 @@ class Peer { this.#name = deviceInfo.name this.#log('received deviceInfo %o', deviceInfo) } - async #assertConnected() { - await this.#connected.promise + #assertConnected() { if (this.#state === 'connected' && !this.#channel.closed) return /* c8 ignore next */ throw new PeerDisconnectedError() // TODO: report error - this should not happen @@ -197,13 +212,17 @@ class Peer { /** @extends {TypedEmitter} */ export class LocalPeers extends TypedEmitter { - /** @type {Map} */ + /** @type {Map>} */ #peers = new Map() + /** @type {Set} */ + #lastEmitterPeers = new Set() /** @type {Set>} */ #opening = new Set() static InviteResponse = InviteResponse_Decision #l + /** @type {Set} */ + #attached = new Set() /** * @@ -215,6 +234,14 @@ export class LocalPeers extends TypedEmitter { this.#l = Logger.create('localPeers', logger) } + get peers() { + const connectedPeerInfos = [] + for (const { info } of this.#getPeers()) { + connectedPeerInfos.push(info) + } + return connectedPeerInfos + } + /** * Invite a peer to a project. Resolves with the response from the invitee: * one of "ACCEPT", "REJECT", or "ALREADY" (already on project) @@ -228,9 +255,8 @@ export class LocalPeers extends TypedEmitter { * @returns {Promise} */ async invite(peerId, { timeout, ...invite }) { - await Promise.all(this.#opening) - const peer = this.#peers.get(peerId) - if (!peer) throw new UnknownPeerError('Unknown peer ' + peerId) + await this.#waitForPendingConnections() + const peer = await this.#getPeerByDeviceId(peerId) /** @type {Promise} */ return new Promise((origResolve, origReject) => { const projectId = keyToId(invite.projectKey) @@ -251,7 +277,11 @@ export class LocalPeers extends TypedEmitter { origReject(new TimeoutError(`No response after ${timeout}ms`)) }, timeout) - peer.sendInvite(invite).catch(origReject) + try { + peer.sendInvite(invite) + } catch (e) { + reject(e) + } /** @type {typeof origResolve} */ function resolve(value) { @@ -275,9 +305,8 @@ export class LocalPeers extends TypedEmitter { * @param {InviteResponse['decision']} options.decision response to invite, one of "ACCEPT", "REJECT", or "ALREADY" (already on project) */ async inviteResponse(peerId, options) { - await Promise.all(this.#opening) - const peer = this.#peers.get(peerId) - if (!peer) throw new UnknownPeerError('Unknown peer ' + peerId) + await this.#waitForPendingConnections() + const peer = await this.#getPeerByDeviceId(peerId) await peer.sendInviteResponse(options) } @@ -287,9 +316,8 @@ export class LocalPeers extends TypedEmitter { * @param {DeviceInfo} deviceInfo device info to send */ async sendDeviceInfo(peerId, deviceInfo) { - await Promise.all(this.#opening) - const peer = this.#peers.get(peerId) - if (!peer) throw new UnknownPeerError('Unknown peer ' + peerId) + await this.#waitForPendingConnections() + const peer = await this.#getPeerByDeviceId(peerId) await peer.sendDeviceInfo(deviceInfo) } @@ -300,13 +328,16 @@ export class LocalPeers extends TypedEmitter { * @returns {import('./types.js').ReplicationStream} */ connect(stream) { - if (!stream.noiseStream) throw new Error('Invalid stream') + const noiseStream = stream.noiseStream + if (!noiseStream) throw new Error('Invalid stream') + const outerStream = noiseStream.rawStream const protomux = - stream.userData && Protomux.isProtomux(stream.userData) - ? stream.userData - : Protomux.from(stream) - stream.userData = protomux - this.#opening.add(stream.opened) + noiseStream.userData && Protomux.isProtomux(noiseStream.userData) + ? noiseStream.userData + : Protomux.from(noiseStream) + noiseStream.userData = protomux + + if (this.#attached.has(protomux)) return outerStream protomux.pair( { protocol: 'hypercore/alpha' }, @@ -316,127 +347,175 @@ export class LocalPeers extends TypedEmitter { discoveryKey, stream.noiseStream.remotePublicKey ) - this.emit('discovery-key', discoveryKey, stream.rawStream) + this.emit('discovery-key', discoveryKey, outerStream) } ) - // No need to connect error handler to stream because Protomux does this, - // and errors are eventually handled by #closePeer + const deferredOpen = pDefer() + this.#opening.add(deferredOpen.promise) + // Called when either the peer opens or disconnects before open + const done = () => { + deferredOpen.resolve() + this.#opening.delete(deferredOpen.promise) + } - // noiseSecretStream.remotePublicKey can be null before the stream has - // opened, so this helped awaits the open - openedNoiseSecretStream(stream).then((stream) => { - this.#opening.delete(stream.opened) - if (stream.destroyed) { - this.#l.log( - 'Opened connection to %h but was already destroyed', - stream.remotePublicKey - ) - return - } - const { remotePublicKey } = stream - - // This is written like this because the protomux uses the index within - // the messages array to define the message id over the wire, so this must - // stay consistent to avoid breaking protocol changes. - /** @type {Parameters[0]['messages']} */ - const messages = new Array(MESSAGES_MAX_ID).fill(undefined) - for (const [type, id] of Object.entries(MESSAGE_TYPES)) { - messages[id] = { - encoding: cenc.raw, - onmessage: this.#handleMessage.bind(this, remotePublicKey, type), - } - } + const makePeer = this.#makePeer.bind(this, protomux, done) - const channel = protomux.createChannel({ - userData: null, - protocol: PROTOCOL_NAME, - messages, - onopen: this.#openPeer.bind(this, remotePublicKey, protomux), - onclose: this.#closePeer.bind(this, remotePublicKey), - }) - channel.open() - - const peerId = keyToId(remotePublicKey) - const existingPeer = this.#peers.get(peerId) - /* c8 ignore next 3 */ - if (existingPeer && existingPeer.info.status !== 'disconnected') { - existingPeer.disconnect() // Should not happen, but in case - } - const peer = new Peer({ - publicKey: remotePublicKey, - channel, - logger: this.#l, - }) - this.#peers.set(peerId, peer) - // Do not emit peers now - will emit when connected + this.#attached.add(protomux) + // This happens when the connected peer opens the channel + protomux.pair( + { protocol: PROTOCOL_NAME }, + // @ts-ignore - need to update protomux types + makePeer + ) + noiseStream.once('close', () => { + this.#attached.delete(protomux) + }) + + noiseStream.opened.then((opened) => { + // Once the noise stream is opened, we attempt to open the channel ourself + // (the peer may have already done this, in which case this is a no-op) + if (opened) makePeer() }) - return stream.rawStream + return outerStream } /** - * @param {Buffer} publicKey - * @param {Protomux} protomux + * @param {Protomux} protomux + * @param {() => void} done */ - #openPeer(publicKey, protomux) { - const peerId = keyToId(publicKey) - const peer = this.#peers.get(peerId) - /* c8 ignore next */ - if (!peer) return // TODO: report error - this should not happen - peer.connect(protomux) - this.#emitPeers() - this.emit('peer-add', /** @type {PeerInfoConnected} */ (peer.info)) + #makePeer(protomux, done) { + // #makePeer is called when the noise stream is opened, but it is also + // called when the connected peer tries to open the channel. We only want + // one channel, so we ignore attempts to create a peer if the channel is + // already open + if (protomux.opened({ protocol: PROTOCOL_NAME })) return done() + + const peerId = keyToId(protomux.stream.remotePublicKey) + + // This is written like this because the protomux uses the index within + // the messages array to define the message id over the wire, so this must + // stay consistent to avoid breaking protocol changes. + /** @type {Parameters[0]['messages']} */ + const messages = new Array(MESSAGES_MAX_ID).fill(undefined) + for (const [type, id] of Object.entries(MESSAGE_TYPES)) { + messages[id] = { + encoding: cenc.raw, + onmessage: this.#handleMessage.bind(this, protomux, type), + } + } + + const channel = protomux.createChannel({ + userData: null, + protocol: PROTOCOL_NAME, + messages, + onopen: () => { + peer.connect() + this.#emitPeers() + done() + }, + onclose: () => { + // TODO: Track reasons for closing + peer.disconnect() + // console.log( + // 'existing', + // [...existingDevicePeers].map( + // ({ info: { protomux, ...rest } }) => rest + // ) + // ) + // We keep disconnected peers around, but not duplicates + if (existingDevicePeers.size > 1) { + // TODO: Decide which existing peer to delete + existingDevicePeers.delete(peer) + } + this.#attached.delete(peer.protomux) + this.#emitPeers() + done() + }, + }) + channel.open() + + const existingDevicePeers = this.#peers.get(peerId) || new Set() + const peer = new Peer({ + peerId, + protomux, + channel, + logger: this.#l, + }) + existingDevicePeers.add(peer) + this.#peers.set(peerId, existingDevicePeers) + // Do not emit peers now - will emit when connected } - /** @param {Buffer} publicKey */ - #closePeer(publicKey) { - const peerId = publicKey.toString('hex') - const peer = this.#peers.get(peerId) - /* c8 ignore next */ - if (!peer) { - this.#l.log('ERROR: Could not close peer %h', publicKey) - return // TODO: report error - this should not happen + /** + * @param {Protomux} protomux + */ + #getPeerByProtomux(protomux) { + // We could also index peers by protomux to avoid this, but that would mean + // we need to keep around protomux references for closed peers, and we keep + // around closed peers for the lifecycle of the app + const peerId = keyToId(protomux.stream.remotePublicKey) + // We could have more than one connection to the same peer + const devicePeers = this.#peers.get(peerId) + /** @type {Peer | undefined} */ + let peer + for (const devicePeer of devicePeers || []) { + if (devicePeer.protomux === protomux) { + peer = devicePeer + } } - // No-op if no change in state - /* c8 ignore next */ - if (peer.info.status === 'disconnected') return - // TODO: Track reasons for closing - peer.disconnect() - this.#emitPeers() + return peer } - get peers() { - return /** @type {PeerInfo[]} */ ( - [...this.#peers.values()] - .map((peer) => peer.info) - // A peer is only 'connecting' for a single tick, so to avoid complex - // async code around sending messages we don't expose 'connecting' peers - .filter((peerInfo) => peerInfo.status !== 'connecting') - ) + #getPeers() { + /** @type {Set} */ + const peers = new Set() + for (const devicePeers of this.#peers.values()) { + const peer = chooseDevicePeer(devicePeers) + // console.log('choose result', peer?.info) + if (peer) peers.add(peer) + } + return peers } #emitPeers() { - this.emit('peers', this.peers) + const currentPeers = this.#getPeers() + const connectedPeerInfos = [] + for (const peer of currentPeers) { + if ( + !this.#lastEmitterPeers.has(peer) && + peer.info.status === 'connected' + ) { + // Any new peers that have 'connected' status + this.emit('peer-add', peer.info) + } + connectedPeerInfos.push(peer.info) + } + if (currentPeers.size > 0 || this.#lastEmitterPeers.size > 0) { + // Don't emit empty array unless somehow it was not empty before + this.emit('peers', connectedPeerInfos) + } + this.#lastEmitterPeers = currentPeers } /** * - * @param {Buffer} peerPublicKey + * @param {Protomux} protomux * @param {keyof typeof MESSAGE_TYPES} type * @param {Buffer} value */ - #handleMessage(peerPublicKey, type, value) { - const peerId = keyToId(peerPublicKey) - const peer = this.#peers.get(peerId) + #handleMessage(protomux, type, value) { + const peer = this.#getPeerByProtomux(protomux) /* c8 ignore next */ if (!peer) return // TODO: report error - this should not happen switch (type) { case 'Invite': { const invite = Invite.decode(value) assertInviteHasKeys(invite) + const peerId = keyToId(protomux.stream.remotePublicKey) this.emit('invite', peerId, invite) - this.#l.log('Invite from %h for %h', peerPublicKey, invite.projectKey) + this.#l.log('Invite from %S for %h', peerId, invite.projectKey) break } case 'InviteResponse': { @@ -452,7 +531,7 @@ export class LocalPeers extends TypedEmitter { } this.#l.log( 'Invite response from %h for %h: %s', - peerPublicKey, + protomux.stream.remotePublicKey, response.projectKey, response.decision ) @@ -474,16 +553,50 @@ export class LocalPeers extends TypedEmitter { } } } -} -export class TimeoutError extends Error { - /** @param {string} [message] */ - constructor(message) { - super(message) - this.name = 'TimeoutError' + /** + * Wait for any connections that are currently opening + */ + #waitForPendingConnections() { + return pTimeout(Promise.all(this.#opening), { milliseconds: SEND_TIMEOUT }) + } + + /** + * Get a peer by deviceId. We can have more than one connection per device, in + * which case we wait for deduplication. Also waits for a peer to be connected + * + * @param {string} deviceId + * @returns {Promise} + */ + async #getPeerByDeviceId(deviceId) { + const devicePeers = this.#peers.get(deviceId) + if (!devicePeers || devicePeers.size === 0) { + throw new UnknownPeerError('Unknown peer ' + deviceId.slice(0, 7)) + } + const peer = chooseDevicePeer(devicePeers) + if (peer) return peer + return new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + this.off('peers', onPeers) + reject(new UnknownPeerError('Unknown peer ' + deviceId.slice(0, 7))) + }, DEDUPE_TIMEOUT) + + this.on('peers', onPeers) + + function onPeers() { + if (!devicePeers) return // Not possible, but let's keep TS happy + const peer = chooseDevicePeer(devicePeers) + if (!peer) return + clearTimeout(timeoutId) + this.off('peers', onPeers) + resolve(peer) + } + }) } } +export { TimeoutError } + export class UnknownPeerError extends Error { /** @param {string} [message] */ constructor(message) { @@ -500,6 +613,14 @@ export class PeerDisconnectedError extends Error { } } +export class PeerFailedConnectionError extends Error { + /** @param {string} [message] */ + constructor(message) { + super(message) + this.name = 'PeerFailedConnectionError' + } +} + /** * * @param {Invite} invite @@ -510,3 +631,50 @@ function assertInviteHasKeys(invite) { throw new Error('Invite is missing auth core encryption key') } } + +function noop() {} + +/** + * We can temporarily have more than 1 peer for a device while connections are + * deduplicating. We don't expose these duplicate connections until only one + * connection exists per device, however if somehow we end up with more than one + * connection with a peer and it is not deduplicated, then we expose the oldest + * connection, or the most recent disconnect. + * + * @param {Set} devicePeers + * @returns {undefined | Peer & { info: PeerInfoConnected | PeerInfoDisconnected }} + */ +function chooseDevicePeer(devicePeers) { + // console.log( + // 'chooseDevicePeer', + // [...devicePeers].map(({ info: { protomux, ...rest } }) => rest) + // ) + if (devicePeers.size === 0) return + let [pick] = devicePeers + if (devicePeers.size > 1) { + for (const peer of devicePeers) { + // If one of the peers for a device is connecting, skip - we'll wait + // until it's connected before returning it. + if (peer.info.status === 'connecting') return + if (peer.info.status === 'connected') { + if (pick.info.status !== 'connected') { + // Always expose the connected peer if there is one + pick = peer + } else if (peer.info.connectedAt < pick.info.connectedAt) { + // If more than one peer is connected, pick the one connected for the longest time + pick = peer + } + } else if ( + pick.info.status === 'disconnected' && + peer.info.disconnectedAt > pick.info.disconnectedAt + ) { + // If all peers are disconnected, pick the most recently disconnected + pick = peer + } + } + } + // Don't expose peers that are connecting, wait until they have connected (or disconnected) + if (pick.info.status === 'connecting') return + // @ts-ignore + return pick +} From bc5e8d64541f7027312adcf9ab41c7c32d0596cf Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 15:01:01 +0900 Subject: [PATCH 39/69] fix stream close before channel open --- src/local-peers.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index ba20da8d9..530c1e7db 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -131,7 +131,7 @@ class Peer { } connect() { - /* c8 ignore next 3 */ + /* c8 ignore next 4 */ if (this.#state !== 'connecting') { this.#log('ERROR: tried to connect but state was %s', this.#state) return // TODO: report error - this should not happen @@ -144,7 +144,7 @@ class Peer { disconnect() { // @ts-ignore - easier to ignore this than handle this for TS - avoids holding a reference to old Protomux instances this.#protomux = undefined - /* c8 ignore next */ + /* c8 ignore next 4 */ if (this.#state === 'disconnected') { this.#log('ERROR: tried to disconnect but was already disconnected') return @@ -370,6 +370,7 @@ export class LocalPeers extends TypedEmitter { ) noiseStream.once('close', () => { this.#attached.delete(protomux) + done() }) noiseStream.opened.then((opened) => { From c1b473c1322336b2168adc5c9e940ba910767edd Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 15:01:34 +0900 Subject: [PATCH 40/69] send invite to non-existent peer --- tests/local-peers.js | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/tests/local-peers.js b/tests/local-peers.js index dc704f8a4..bced8f1b4 100644 --- a/tests/local-peers.js +++ b/tests/local-peers.js @@ -388,8 +388,6 @@ test('Default: invites do not timeout', async (t) => { }) test('Invite timeout', async (t) => { - const clock = FakeTimers.install({ shouldAdvanceTime: true }) - t.teardown(() => clock.uninstall()) t.plan(1) const r1 = new LocalPeers() @@ -399,20 +397,35 @@ test('Invite timeout', async (t) => { r1.once('peers', async (peers) => { t.exception( - r1.invite(peers[0].deviceId, { - projectKey, - timeout: 1000, - encryptionKeys: { auth: randomBytes(32) }, - }), + () => + r1.invite(peers[0].deviceId, { + projectKey, + timeout: 1000, + encryptionKeys: { auth: randomBytes(32) }, + }), TimeoutError ) - // Not working right now, because of the new async code - clock.tick(5001) }) replicate(r1, r2) }) +test('Send invite to non-existent peer', async (t) => { + const r1 = new LocalPeers() + const projectKey = Buffer.allocUnsafe(32).fill(0) + const deviceId = Buffer.allocUnsafe(32).fill(0).toString('hex') + + await t.exception( + () => + r1.invite(deviceId, { + projectKey, + timeout: 1000, + encryptionKeys: { auth: randomBytes(32) }, + }), + UnknownPeerError + ) +}) + test('Reconnect peer and send invite', async (t) => { const r1 = new LocalPeers() const r2 = new LocalPeers() From 07cac79b3e2d8ec3a32f0e13034aed1564186010 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 17:10:34 +0900 Subject: [PATCH 41/69] fixed fake timers implementation for tests --- tests/local-peers.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/local-peers.js b/tests/local-peers.js index bced8f1b4..04405f8dd 100644 --- a/tests/local-peers.js +++ b/tests/local-peers.js @@ -388,6 +388,8 @@ test('Default: invites do not timeout', async (t) => { }) test('Invite timeout', async (t) => { + const clock = FakeTimers.install({ shouldAdvanceTime: true }) + t.teardown(() => clock.uninstall()) t.plan(1) const r1 = new LocalPeers() @@ -400,11 +402,12 @@ test('Invite timeout', async (t) => { () => r1.invite(peers[0].deviceId, { projectKey, - timeout: 1000, + timeout: 5000, encryptionKeys: { auth: randomBytes(32) }, }), TimeoutError ) + clock.tickAsync(5005) }) replicate(r1, r2) From 3407e0c96b6cd7c93e321e414164fbc276677f71 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 17:10:45 +0900 Subject: [PATCH 42/69] new tests for duplicate connections --- tests/local-peers.js | 92 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tests/local-peers.js b/tests/local-peers.js index 04405f8dd..084b7e8ba 100644 --- a/tests/local-peers.js +++ b/tests/local-peers.js @@ -69,6 +69,98 @@ test('Send invite immediately', async (t) => { t.is(await responsePromise, LocalPeers.InviteResponse.ACCEPT) }) +test('Send invite, duplicate connections', async (t) => { + const r1 = new LocalPeers() + const r2 = new LocalPeers() + + const invite = { + projectKey: Buffer.allocUnsafe(32).fill(0), + encryptionKeys: { auth: randomBytes(32) }, + } + + const kp1 = NoiseSecretStream.keyPair() + const kp2 = NoiseSecretStream.keyPair() + + const destroy1 = replicate(r1, r2, { kp1, kp2 }) + const [peers1] = await once(r1, 'peers') + const destroy2 = replicate(r1, r2, { kp1, kp2 }) + const [peers2] = await once(r1, 'peers') + + t.is(peers1.length, 1) + t.is(peers2.length, 1) + t.is(peers1[0].connectedAt, peers2[0].connectedAt, 'first connected is used') + + { + const responsePromise = r1.invite(peers1[0].deviceId, invite) + const [peerId, receivedInvite] = await once(r2, 'invite') + t.alike(receivedInvite, invite) + + r2.inviteResponse(peerId, { + projectKey: receivedInvite.projectKey, + decision: LocalPeers.InviteResponse.ACCEPT, + }) + + t.is(await responsePromise, LocalPeers.InviteResponse.ACCEPT) + } + + destroy1() + const [peers3] = await once(r1, 'peers') + + t.is(peers3.length, 1) + t.ok( + peers3[0].connectedAt > peers1[0].connectedAt, + 'later connected peer is not used' + ) + + { + const responsePromise = r1.invite(peers1[0].deviceId, invite) + const [peerId, receivedInvite] = await once(r2, 'invite') + t.alike(receivedInvite, invite) + + r2.inviteResponse(peerId, { + projectKey: receivedInvite.projectKey, + decision: LocalPeers.InviteResponse.ACCEPT, + }) + + t.is(await responsePromise, LocalPeers.InviteResponse.ACCEPT) + } + + const now = Date.now() + destroy2() + const [peers4] = await once(r1, 'peers') + t.is(peers4.length, 1) + t.is(peers4[0].status, 'disconnected') + t.ok(peers4[0].disconnectedAt >= now, 'most recently disconnected exposed') +}) + +test('Duplicate connections with immediate disconnect', async (t) => { + const r1 = new LocalPeers() + const r2 = new LocalPeers() + + const invite = { + projectKey: Buffer.allocUnsafe(32).fill(0), + encryptionKeys: { auth: randomBytes(32) }, + } + + const kp1 = NoiseSecretStream.keyPair() + const kp2 = NoiseSecretStream.keyPair() + + replicate(r1, r2, { kp1, kp2 }) + const destroy2 = replicate(r1, r2, { kp1, kp2 }) + destroy2() + + const responsePromise = r1.invite(kp2.publicKey.toString('hex'), invite) + const [peerId, receivedInvite] = await once(r2, 'invite') + t.alike(receivedInvite, invite) + + r2.inviteResponse(peerId, { + projectKey: receivedInvite.projectKey, + decision: LocalPeers.InviteResponse.ACCEPT, + }) + + t.is(await responsePromise, LocalPeers.InviteResponse.ACCEPT) +}) + test('Send invite and reject', async (t) => { t.plan(3) const r1 = new LocalPeers() From 8387f84f3b24c74effc8391284edb1c6d2b9f7a4 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 17:30:41 +0900 Subject: [PATCH 43/69] cleanup and small fix --- src/local-peers.js | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index 530c1e7db..03441a877 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -330,14 +330,13 @@ export class LocalPeers extends TypedEmitter { connect(stream) { const noiseStream = stream.noiseStream if (!noiseStream) throw new Error('Invalid stream') - const outerStream = noiseStream.rawStream const protomux = noiseStream.userData && Protomux.isProtomux(noiseStream.userData) ? noiseStream.userData : Protomux.from(noiseStream) noiseStream.userData = protomux - if (this.#attached.has(protomux)) return outerStream + if (this.#attached.has(protomux)) return stream protomux.pair( { protocol: 'hypercore/alpha' }, @@ -347,7 +346,7 @@ export class LocalPeers extends TypedEmitter { discoveryKey, stream.noiseStream.remotePublicKey ) - this.emit('discovery-key', discoveryKey, outerStream) + this.emit('discovery-key', discoveryKey, stream) } ) @@ -379,7 +378,7 @@ export class LocalPeers extends TypedEmitter { if (opened) makePeer() }) - return outerStream + return stream } /** @@ -419,12 +418,6 @@ export class LocalPeers extends TypedEmitter { onclose: () => { // TODO: Track reasons for closing peer.disconnect() - // console.log( - // 'existing', - // [...existingDevicePeers].map( - // ({ info: { protomux, ...rest } }) => rest - // ) - // ) // We keep disconnected peers around, but not duplicates if (existingDevicePeers.size > 1) { // TODO: Decide which existing peer to delete @@ -474,7 +467,6 @@ export class LocalPeers extends TypedEmitter { const peers = new Set() for (const devicePeers of this.#peers.values()) { const peer = chooseDevicePeer(devicePeers) - // console.log('choose result', peer?.info) if (peer) peers.add(peer) } return peers @@ -646,10 +638,6 @@ function noop() {} * @returns {undefined | Peer & { info: PeerInfoConnected | PeerInfoDisconnected }} */ function chooseDevicePeer(devicePeers) { - // console.log( - // 'chooseDevicePeer', - // [...devicePeers].map(({ info: { protomux, ...rest } }) => rest) - // ) if (devicePeers.size === 0) return let [pick] = devicePeers if (devicePeers.size > 1) { From 6a886b8be081af77d82f26adbf55bb7224d12a4a Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 17:55:41 +0900 Subject: [PATCH 44/69] Better state debug logging --- src/logger.js | 28 +++++++++++++++++++++++++--- src/sync/peer-sync-controller.js | 2 +- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/src/logger.js b/src/logger.js index 356e6cc35..0c82fbc41 100644 --- a/src/logger.js +++ b/src/logger.js @@ -1,23 +1,45 @@ import createDebug from 'debug' import { discoveryKey } from 'hypercore-crypto' +import mapObject from 'map-obj' +import util from 'util' const TRIM = 7 -createDebug.formatters.h = (v) => { +createDebug.formatters.h = function (v) { if (!Buffer.isBuffer(v)) return '[undefined]' return v.toString('hex').slice(0, TRIM) } -createDebug.formatters.S = (v) => { +createDebug.formatters.S = function (v) { if (typeof v !== 'string') return '[undefined]' return v.slice(0, 7) } -createDebug.formatters.k = (v) => { +createDebug.formatters.k = function (v) { if (!Buffer.isBuffer(v)) return '[undefined]' return discoveryKey(v).toString('hex').slice(0, TRIM) } +/** + * @param {import('./sync/sync-state.js').State} v + * @this {any} */ +createDebug.formatters.X = function (v) { + const mapped = mapObject(v, (k, v) => [ + k, + mapObject(v, (k, v) => { + if (k === 'remoteStates') + return [k, mapObject(v, (k, v) => [k.slice(0, 7), v])] + return [k, v] + }), + ]) + return util.inspect(mapped, { + colors: true, + depth: 10, + compact: 6, + breakLength: 90, + }) +} + const counts = new Map() export class Logger { diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index c076a720a..ddcd42881 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -142,7 +142,7 @@ export class PeerSyncController { const localState = mapObject(state, (ns, nsState) => { return [ns, nsState.localState] }) - this.#log('state %O', state) + this.#log('state %X', state) // Map of which namespaces have received new data since last sync change const didUpdate = mapObject(state, (ns) => { From d50fcea7687ee4507947aadcb503e88cf877398b Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 20:50:17 +0900 Subject: [PATCH 45/69] chain of invites test --- test-e2e/manager-invite.js | 61 +++++++++++++++++++++++++++++++++++++- test-e2e/utils-new.js | 6 ++-- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index 7cfea1ab8..f9d4081e2 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -1,5 +1,5 @@ import { test } from 'brittle' -import { MEMBER_ROLE_ID } from '../src/capabilities.js' +import { COORDINATOR_ROLE_ID, MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' import { once } from 'node:events' import { @@ -63,6 +63,55 @@ test('member invite accepted', async (t) => { await disconnectPeers([creator, joiner]) }) +test('chain of invites', async (t) => { + const managers = await createManagers(6) + const [creator, ...joiners] = managers + connectPeers(managers) + await waitForPeers(managers) + + const createdProjectId = await creator.createProject({ name: 'Mapeo' }) + + let invitor = creator + for (const joiner of joiners) { + const invitorProject = await invitor.getProject(createdProjectId) + const responsePromise = invitorProject.$member.invite(joiner.deviceId, { + roleId: COORDINATOR_ROLE_ID, + }) + const [invite] = await once(joiner.invite, 'invite-received') + await joiner.invite.accept(invite.projectId) + t.is( + await responsePromise, + InviteResponse_Decision.ACCEPT, + 'correct invite response' + ) + } + + /// After invite flow has completed... + + const creatorProject = await creator.getProject(createdProjectId) + const expectedProjectSettings = await creatorProject.$getProjectSettings() + const expectedMembers = await creatorProject.$member.getMany() + + for (const joiner of joiners) { + const joinerProject = await joiner.getProject(createdProjectId) + + t.alike( + await joinerProject.$getProjectSettings(), + expectedProjectSettings, + 'Project settings match' + ) + + const joinerMembers = await joinerProject.$member.getMany() + t.alike( + joinerMembers.sort(memberSort), + expectedMembers.sort(memberSort), + 'Project members match' + ) + } + + await disconnectPeers(managers) +}) + test('member invite rejected', async (t) => { const [creator, joiner] = await createManagers(2) connectPeers([creator, joiner]) @@ -111,3 +160,13 @@ test('member invite rejected', async (t) => { await disconnectPeers([creator, joiner]) }) + +/** + * @param {import('../src/member-api.js').MemberInfo} a + * @param {import('../src/member-api.js').MemberInfo} b + */ +function memberSort(a, b) { + if (a.deviceId < b.deviceId) return -1 + if (a.deviceId > b.deviceId) return 1 + return 0 +} diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js index a5d7e65bf..f6fbe3b9d 100644 --- a/test-e2e/utils-new.js +++ b/test-e2e/utils-new.js @@ -5,7 +5,7 @@ import RAM from 'random-access-memory' import { MapeoManager } from '../src/index.js' /** - * @param {MapeoManager[]} managers + * @param {readonly MapeoManager[]} managers */ export async function disconnectPeers(managers) { return Promise.all( @@ -16,7 +16,7 @@ export async function disconnectPeers(managers) { } /** - * @param {MapeoManager[]} managers + * @param {readonly MapeoManager[]} managers */ export function connectPeers(managers, { discovery = true } = {}) { if (discovery) { @@ -31,7 +31,7 @@ export function connectPeers(managers, { discovery = true } = {}) { /** * Waits for all manager instances to be connected to each other * - * @param {MapeoManager[]} managers + * @param {readonly MapeoManager[]} managers */ export async function waitForPeers(managers) { const peerCounts = Array(managers.length).fill(0) From 5bf194163ce13fd4f1a925d8c078ee9feb7673c4 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Tue, 14 Nov 2023 22:22:08 +0900 Subject: [PATCH 46/69] fix max listeners and add skipped test --- src/core-manager/index.js | 4 ++++ src/sync/sync-api.js | 1 + test-e2e/manager-invite.js | 40 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index e57b4adf3..803a6ea02 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -84,6 +84,8 @@ export class CoreManager extends TypedEmitter { !projectSecretKey || projectSecretKey.length === 64, 'project owner core secret key must be 64-byte buffer' ) + // Each peer will attach a listener, so max listeners is max attached peers + this.setMaxListeners(0) this.#l = Logger.create('coreManager', logger) const primaryKey = keyManager.getDerivedKey('primaryKey', projectKey) this.#deviceId = keyManager.getIdentityKeypair().publicKey.toString('hex') @@ -271,6 +273,8 @@ export class CoreManager extends TypedEmitter { // Starts live download of core immediately sparse: namespace === 'blob', }) + // Every peer adds a listener, so could have many peers + core.setMaxListeners(0) // @ts-ignore - ensure key is defined before hypercore is ready core.key = key this.#coreIndex.add({ core, key, namespace, writer }) diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index 45629450a..e3b56c161 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -39,6 +39,7 @@ export class SyncApi extends TypedEmitter { this.#coreManager = coreManager this.#capabilities = capabilities this.syncState = new SyncState({ coreManager, throttleMs }) + this.syncState.setMaxListeners(0) this.syncState.on('state', this.emit.bind(this, 'sync-state')) this.#coreManager.creatorCore.on('peer-add', this.#handlePeerAdd) diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index f9d4081e2..daef54f05 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -1,4 +1,4 @@ -import { test } from 'brittle' +import { test, skip } from 'brittle' import { COORDINATOR_ROLE_ID, MEMBER_ROLE_ID } from '../src/capabilities.js' import { InviteResponse_Decision } from '../src/generated/rpc.js' import { once } from 'node:events' @@ -64,7 +64,7 @@ test('member invite accepted', async (t) => { }) test('chain of invites', async (t) => { - const managers = await createManagers(6) + const managers = await createManagers(10) const [creator, ...joiners] = managers connectPeers(managers) await waitForPeers(managers) @@ -112,6 +112,42 @@ test('chain of invites', async (t) => { await disconnectPeers(managers) }) +// Needs fix to inviteApi to check capabilities before sending invite +skip("member can't invite", async (t) => { + const managers = await createManagers(3) + const [creator, member, joiner] = managers + connectPeers(managers) + await waitForPeers(managers) + + const createdProjectId = await creator.createProject({ name: 'Mapeo' }) + const creatorProject = await creator.getProject(createdProjectId) + + const responsePromise = creatorProject.$member.invite(member.deviceId, { + roleId: MEMBER_ROLE_ID, + }) + const [invite] = await once(member.invite, 'invite-received') + await member.invite.accept(invite.projectId) + await responsePromise + + /// After invite flow has completed... + + const memberProject = await member.getProject(createdProjectId) + + t.alike( + await memberProject.$getProjectSettings(), + await creatorProject.$getProjectSettings(), + 'Project settings match' + ) + + const exceptionPromise = t.exception(() => + memberProject.$member.invite(joiner.deviceId, { roleId: MEMBER_ROLE_ID }) + ) + joiner.invite.once('invite-received', () => t.fail('should not send invite')) + await exceptionPromise + + await disconnectPeers(managers) +}) + test('member invite rejected', async (t) => { const [creator, joiner] = await createManagers(2) connectPeers([creator, joiner]) From 34d2d3467afd99360509f8582c1e5b3e6baa8b56 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 16 Nov 2023 12:38:01 +0900 Subject: [PATCH 47/69] fix: only request a core key from one peer Reduces the number of duplicate requests for the same keys. --- package-lock.json | 5 ++- src/core-manager/index.js | 84 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 85 insertions(+), 4 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5e30ab7cd..bd6f02c41 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8297,8 +8297,9 @@ "license": "ISC" }, "node_modules/xache": { - "version": "1.1.0", - "license": "MIT" + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/xache/-/xache-1.2.0.tgz", + "integrity": "sha512-AiWCGTrnsh//rrbJt8DLbDkDW8eLp1Ktkq0nTWzpE+FKCY35oeqLjtz+LNb4abMnjfTgL0ZBaSwzhgzan1ocEw==" }, "node_modules/xsalsa20": { "version": "1.2.0", diff --git a/src/core-manager/index.js b/src/core-manager/index.js index 803a6ea02..4f52a0c2f 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -6,6 +6,8 @@ import { HaveExtension, ProjectExtension } from '../generated/extensions.js' import { CoreIndex } from './core-index.js' import * as rle from './bitfield-rle.js' import { Logger } from '../logger.js' +import { keyToId } from '../utils.js' +import { discoveryKey } from 'hypercore-crypto' // WARNING: Changing these will break things for existing apps, since namespaces // are used for key derivation @@ -51,6 +53,12 @@ export class CoreManager extends TypedEmitter { #haveExtension #deviceId #l + /** + * We use this to reduce network traffic caused by requesting the same key + * from multiple clients. + * TODO: Remove items from this set after a max age + */ + #keyRequests = new TrackedKeyRequests() static get namespaces() { return NAMESPACES @@ -157,6 +165,12 @@ export class CoreManager extends TypedEmitter { this.#creatorCore.on('peer-add', (peer) => { this.#sendHaves(peer) }) + this.#creatorCore.on('peer-remove', (peer) => { + // When a peer is removed we clean up any unanswered key requests, so that + // we will request from a different peer, and to avoid the tracking of key + // requests growing without bounds. + this.#keyRequests.deleteByPeerKey(peer.remotePublicKey) + }) this.#ready = Promise.all( [...this.#coreIndex].map(({ core }) => core.ready()) @@ -233,6 +247,7 @@ export class CoreManager extends TypedEmitter { */ async close() { this.#state = 'closing' + this.#keyRequests.clear() const promises = [] for (const { core } of this.#coreIndex) { promises.push(core.close()) @@ -342,6 +357,13 @@ export class CoreManager extends TypedEmitter { ) return } + // Only request a key once, e.g. from the peer we first receive it from (we + // can assume that a peer must have the key if we see the discovery key in + // the protomux). This is necessary to reduce network traffic for many newly + // connected peers - otherwise duplicate requests will be sent to every peer + if (this.#keyRequests.has(discoveryKey)) return + this.#keyRequests.set(discoveryKey, peerKey) + this.#l.log( 'Requesting core key for discovery key %h from peer %h', discoveryKey, @@ -361,8 +383,7 @@ export class CoreManager extends TypedEmitter { const message = ProjectExtension.create() let hasKeys = false for (const discoveryKey of wantCoreKeys) { - const discoveryId = discoveryKey.toString('hex') - const coreRecord = this.#coreIndex.getByDiscoveryId(discoveryId) + const coreRecord = this.getCoreByDiscoveryKey(discoveryKey) if (!coreRecord) continue message[`${coreRecord.namespace}CoreKeys`].push(coreRecord.key) hasKeys = true @@ -374,6 +395,7 @@ export class CoreManager extends TypedEmitter { for (const coreKey of coreKeys[`${namespace}CoreKeys`]) { // Use public method - these must be persisted (private method defaults to persisted=false) this.addCore(coreKey, namespace) + this.#keyRequests.deleteByDiscoveryKey(discoveryKey(coreKey)) } } } @@ -472,3 +494,61 @@ const HaveExtensionCodec = { } }, } + +class TrackedKeyRequests { + /** @type {Map} */ + #byDiscoveryId = new Map() + /** @type {Map>} */ + #byPeerId = new Map() + + /** + * @param {Buffer} discoveryKey + * @param {Buffer} peerKey + */ + set(discoveryKey, peerKey) { + const discoveryId = keyToId(discoveryKey) + const peerId = keyToId(peerKey) + const existingForPeer = this.#byPeerId.get(peerId) || new Set() + this.#byDiscoveryId.set(discoveryId, peerId) + existingForPeer.add(discoveryId) + this.#byPeerId.set(peerId, existingForPeer) + return this + } + /** + * @param {Buffer} discoveryKey + */ + has(discoveryKey) { + const discoveryId = keyToId(discoveryKey) + return this.#byDiscoveryId.has(discoveryId) + } + /** + * @param {Buffer} discoveryKey + */ + deleteByDiscoveryKey(discoveryKey) { + const discoveryId = keyToId(discoveryKey) + const peerId = this.#byDiscoveryId.get(discoveryId) + if (!peerId) return false + this.#byDiscoveryId.delete(discoveryId) + const existingForPeer = this.#byPeerId.get(peerId) + if (existingForPeer) { + existingForPeer.delete(discoveryId) + } + return true + } + /** + * @param {Buffer} peerKey + */ + deleteByPeerKey(peerKey) { + const peerId = keyToId(peerKey) + const existingForPeer = this.#byPeerId.get(peerId) + if (!existingForPeer) return + for (const discoveryId of existingForPeer) { + this.#byDiscoveryId.delete(discoveryId) + } + this.#byPeerId.delete(peerId) + } + clear() { + this.#byDiscoveryId.clear() + this.#byPeerId.clear() + } +} From 158ac7c5fdc530a41382881ea56ee5c4a34cbc74 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 16 Nov 2023 13:26:18 +0900 Subject: [PATCH 48/69] cleanup members tests with new helprs --- test-e2e/members.js | 161 +++++++++++++++--------------------------- test-e2e/utils-new.js | 50 ++++++++++++- 2 files changed, 105 insertions(+), 106 deletions(-) diff --git a/test-e2e/members.js b/test-e2e/members.js index 5a951c852..23998c805 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -1,22 +1,25 @@ +// @ts-check import { test } from 'brittle' -import RAM from 'random-access-memory' -import { KeyManager } from '@mapeo/crypto' -import pDefer from 'p-defer' import { randomBytes } from 'crypto' -import { MapeoManager, kRPC } from '../src/mapeo-manager.js' import { CREATOR_CAPABILITIES, DEFAULT_CAPABILITIES, MEMBER_ROLE_ID, NO_ROLE_CAPABILITIES, } from '../src/capabilities.js' -import { replicate } from '../tests/helpers/local-peers.js' +import { + connectPeers, + createManagers, + disconnectPeers, + invite, + waitForPeers, +} from './utils-new.js' test('getting yourself after creating project', async (t) => { - const { manager } = setup() + const [manager] = await createManagers(1) - await manager.setDeviceInfo({ name: 'mapeo' }) + const deviceInfo = await manager.getDeviceInfo() const project = await manager.getProject(await manager.createProject()) await project.ready() @@ -26,7 +29,7 @@ test('getting yourself after creating project', async (t) => { me, { deviceId: project.deviceId, - name: 'mapeo', + name: deviceInfo.name, capabilities: CREATOR_CAPABILITIES, }, 'has expected member info with creator capabilities' @@ -39,17 +42,17 @@ test('getting yourself after creating project', async (t) => { members[0], { deviceId: project.deviceId, - name: 'mapeo', + name: deviceInfo.name, capabilities: CREATOR_CAPABILITIES, }, 'has expected member info with creator capabilities' ) }) -test('getting yourself after being invited to project (but not yet synced)', async (t) => { - const { manager } = setup() +test('getting yourself after adding project (but not yet synced)', async (t) => { + const [manager] = await createManagers(1) - await manager.setDeviceInfo({ name: 'mapeo' }) + const deviceInfo = await manager.getDeviceInfo() const project = await manager.getProject( await manager.addProject( { @@ -67,7 +70,7 @@ test('getting yourself after being invited to project (but not yet synced)', asy me, { deviceId: project.deviceId, - name: 'mapeo', + name: deviceInfo.name, capabilities: NO_ROLE_CAPABILITIES, }, 'has expected member info with no role capabilities' @@ -80,7 +83,7 @@ test('getting yourself after being invited to project (but not yet synced)', asy members[0], { deviceId: project.deviceId, - name: 'mapeo', + name: deviceInfo.name, capabilities: NO_ROLE_CAPABILITIES, }, 'has expected member info with no role capabilities' @@ -88,19 +91,24 @@ test('getting yourself after being invited to project (but not yet synced)', asy }) test('getting invited member after invite rejected', async (t) => { - const { manager, simulateMemberInvite } = setup() + const managers = await createManagers(2) + const [invitor, invitee] = managers + connectPeers(managers) + await waitForPeers(managers) - await manager.setDeviceInfo({ name: 'mapeo' }) - const project = await manager.getProject(await manager.createProject()) + const projectId = await invitor.createProject() + const project = await invitor.getProject(projectId) await project.ready() - const invitedDeviceId = await simulateMemberInvite(project, 'reject', { - deviceInfo: { name: 'member' }, - roleId: MEMBER_ROLE_ID, + await invite({ + invitor, + projectId, + invitees: [invitee], + reject: true, }) await t.exception( - () => project.$member.getById(invitedDeviceId), + () => project.$member.getById(invitee.deviceId), 'invited member cannot be retrieved' ) @@ -108,103 +116,46 @@ test('getting invited member after invite rejected', async (t) => { t.is(members.length, 1) t.absent( - members.find((m) => m.deviceId === invitedDeviceId), + members.find((m) => m.deviceId === invitee.deviceId), 'invited member not found' ) + await disconnectPeers(managers) }) test('getting invited member after invite accepted', async (t) => { - const { manager, simulateMemberInvite } = setup() - - await manager.setDeviceInfo({ name: 'mapeo' }) - const project = await manager.getProject(await manager.createProject()) + const managers = await createManagers(2) + const [invitor, invitee] = managers + connectPeers(managers) + await waitForPeers(managers) + + const { name: inviteeName } = await invitee.getDeviceInfo() + const projectId = await invitor.createProject() + const project = await invitor.getProject(projectId) await project.ready() - const invitedDeviceId = await simulateMemberInvite(project, 'accept', { - deviceInfo: { name: 'member' }, + await invite({ + invitor, + projectId, + invitees: [invitee], roleId: MEMBER_ROLE_ID, }) - // Before syncing - { - const invitedMember = await project.$member.getById(invitedDeviceId) - - t.alike( - invitedMember, - { - deviceId: invitedDeviceId, - capabilities: DEFAULT_CAPABILITIES[MEMBER_ROLE_ID], - }, - 'has expected member info with member capabilities' - ) - } - - { - const members = await project.$member.getMany() + const members = await project.$member.getMany() - t.is(members.length, 2) + t.is(members.length, 2) - const invitedMember = members.find((m) => m.deviceId === invitedDeviceId) + const invitedMember = members.find((m) => m.deviceId === invitee.deviceId) - t.alike( - invitedMember, - { - deviceId: invitedDeviceId, - capabilities: DEFAULT_CAPABILITIES[MEMBER_ROLE_ID], - }, - 'has expected member info with member capabilities' - ) - } + t.alike( + invitedMember, + { + deviceId: invitee.deviceId, + name: inviteeName, + capabilities: DEFAULT_CAPABILITIES[MEMBER_ROLE_ID], + }, + 'has expected member info with member capabilities' + ) // TODO: Test that device info of invited member can be read from invitor after syncing + await disconnectPeers(managers) }) - -function setup() { - const manager = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) - - /** - * - * @param {import('../src/mapeo-project.js').MapeoProject} project - * @param {'accept' | 'reject'} respondWith - * @param {{ deviceInfo: import('../src/generated/rpc.js').DeviceInfo, roleId: import('../src/capabilities.js').RoleId }} mocked - * - */ - async function simulateMemberInvite( - project, - respondWith, - { deviceInfo, roleId } - ) { - /** @type {import('p-defer').DeferredPromise} */ - const deferred = pDefer() - - const otherManager = new MapeoManager({ - rootKey: KeyManager.generateRootKey(), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) - - await otherManager.setDeviceInfo(deviceInfo) - - otherManager.invite.on('invite-received', ({ projectId }) => { - otherManager.invite[respondWith](projectId).catch(deferred.reject) - }) - - manager[kRPC].on('peers', (peers) => { - const deviceId = peers[0].deviceId - project.$member - .invite(deviceId, { roleId }) - .then(() => deferred.resolve(deviceId)) - .catch(deferred.reject) - }) - - replicate(manager[kRPC], otherManager[kRPC]) - - return deferred.promise - } - - return { manager, simulateMemberInvite } -} diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js index f6fbe3b9d..37e8aee4c 100644 --- a/test-e2e/utils-new.js +++ b/test-e2e/utils-new.js @@ -3,6 +3,9 @@ import sodium from 'sodium-universal' import RAM from 'random-access-memory' import { MapeoManager } from '../src/index.js' +import { kRPC } from '../src/mapeo-manager.js' +import { MEMBER_ROLE_ID } from '../src/capabilities.js' +import { once } from 'node:events' /** * @param {readonly MapeoManager[]} managers @@ -28,15 +31,60 @@ export function connectPeers(managers, { discovery = true } = {}) { } } +/** + * Invite mapeo clients to a project + * + * @param {{ + * invitor: MapeoManager, + * projectId: string, + * invitees: MapeoManager[], + * roleId?: import('../src/capabilities.js').RoleId, + * reject?: boolean + * }} opts + */ +export async function invite({ + invitor, + projectId, + invitees, + roleId = MEMBER_ROLE_ID, + reject = false, +}) { + const invitorProject = await invitor.getProject(projectId) + const promises = [] + + for (const invitee of invitees) { + promises.push( + invitorProject.$member.invite(invitee.deviceId, { + roleId, + }) + ) + promises.push( + once(invitee.invite, 'invite-received').then(([invite]) => { + return reject + ? invitee.invite.reject(invite.projectId) + : invitee.invite.accept(invite.projectId) + }) + ) + } + + await Promise.allSettled(promises) +} + /** * Waits for all manager instances to be connected to each other * * @param {readonly MapeoManager[]} managers */ export async function waitForPeers(managers) { - const peerCounts = Array(managers.length).fill(0) + const peerCounts = managers.map((manager) => { + return manager[kRPC].peers.filter(({ status }) => status === 'connected') + .length + }) const expectedCount = managers.length - 1 return new Promise((res) => { + if (peerCounts.every((v) => v === expectedCount)) { + return res(null) + } for (const [idx, manager] of managers.entries()) { manager.on('local-peers', function onPeers(peers) { const connectedPeerCount = peers.filter( From 687b9bc12e7afc01da87bf040aee71bd47c2b01f Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 16 Nov 2023 13:26:32 +0900 Subject: [PATCH 49/69] wait for project ready when adding --- src/mapeo-manager.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 0243cf9e8..053fe32e6 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -487,6 +487,7 @@ export class MapeoManager extends TypedEmitter { try { // 4. Write device info into project const project = await this.getProject(projectPublicId) + await project.ready() try { const deviceInfo = await this.getDeviceInfo() From e6225fef820c654e53908cb6d92b65592592a8ac Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 16 Nov 2023 13:27:11 +0900 Subject: [PATCH 50/69] only create 4 clients for chain of invites test --- test-e2e/manager-invite.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index daef54f05..2500e1c62 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -64,7 +64,7 @@ test('member invite accepted', async (t) => { }) test('chain of invites', async (t) => { - const managers = await createManagers(10) + const managers = await createManagers(4) const [creator, ...joiners] = managers connectPeers(managers) await waitForPeers(managers) From a912edeb99d96121e5beacee419630b06e3359c2 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Fri, 17 Nov 2023 20:34:39 +0900 Subject: [PATCH 51/69] add e2e sync tests --- package-lock.json | 129 +++++++++++++++++++++++++++- package.json | 2 + src/sync/peer-sync-controller.js | 2 +- src/sync/sync-api.js | 41 ++++++++- test-e2e/project-crud.js | 19 +---- test-e2e/sync.js | 119 ++++++++++++++++++++++++++ test-e2e/utils-new.js | 142 ++++++++++++++++++++++++++++++- 7 files changed, 430 insertions(+), 24 deletions(-) create mode 100644 test-e2e/sync.js diff --git a/package-lock.json b/package-lock.json index bd6f02c41..e51054265 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,6 +55,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", + "@mapeo/mock-data": "github:digidem/mapeo-mock-data#feat/sync-generate", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", @@ -81,6 +82,7 @@ "prettier": "^2.8.8", "random-access-file": "^4.0.4", "random-access-memory": "^6.2.0", + "random-bytes-readable-stream": "^3.0.0", "rimraf": "^5.0.1", "streamx": "^2.15.1", "tempy": "^3.1.0", @@ -531,6 +533,22 @@ "node": "^12.22.0 || ^14.17.0 || >=16.0.0" } }, + "node_modules/@faker-js/faker": { + "version": "8.3.1", + "resolved": "https://registry.npmjs.org/@faker-js/faker/-/faker-8.3.1.tgz", + "integrity": "sha512-FdgpFxY6V6rLZE9mmIBb9hM0xpfvQOSNOLnzolzKwsE1DH+gC7lEKV1p1IbR0lAYyvYd5a4u3qWJzowUkw1bIw==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/fakerjs" + } + ], + "engines": { + "node": "^14.17.0 || ^16.13.0 || >=18.0.0", + "npm": ">=6.14.13" + } + }, "node_modules/@fastify/ajv-compiler": { "version": "3.5.0", "dev": true, @@ -832,6 +850,22 @@ "z32": "^1.0.0" } }, + "node_modules/@mapeo/mock-data": { + "version": "0.0.0", + "resolved": "git+ssh://git@github.com/digidem/mapeo-mock-data.git#6cc68aa1acc6f4b7482a758df7ab00984fa7707d", + "dev": true, + "license": "MIT", + "dependencies": { + "@faker-js/faker": "^8.3.1", + "@mapeo/schema": "^3.0.0-next.11", + "json-schema-faker": "^0.5.3", + "type-fest": "^4.8.0" + }, + "bin": { + "generate-mapeo-data": "bin/generate-mapeo-data.js", + "list-mapeo-schemas": "bin/list-mapeo-schemas.js" + } + }, "node_modules/@mapeo/schema": { "version": "3.0.0-next.13", "resolved": "https://registry.npmjs.org/@mapeo/schema/-/schema-3.0.0-next.13.tgz", @@ -1729,6 +1763,12 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/call-me-maybe": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-me-maybe/-/call-me-maybe-1.0.2.tgz", + "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==", + "dev": true + }, "node_modules/callsites": { "version": "3.1.0", "dev": true, @@ -3433,6 +3473,12 @@ "node": ">=8.0.0" } }, + "node_modules/format-util": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/format-util/-/format-util-1.0.5.tgz", + "integrity": "sha512-varLbTj0e0yVyRpqQhuWV+8hlePAgaoFRhNFj50BNjEIrw1/DphHSObtqwskVCPWNgzwPoQrZAbfa/SBiicNeg==", + "dev": true + }, "node_modules/forwarded": { "version": "0.2.0", "dev": true, @@ -4500,6 +4546,53 @@ "dev": true, "license": "MIT" }, + "node_modules/json-schema-faker": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/json-schema-faker/-/json-schema-faker-0.5.3.tgz", + "integrity": "sha512-BeIrR0+YSrTbAR9dOMnjbFl1MvHyXnq+Wpdw1FpWZDHWKLzK229hZ5huyPcmzFUfVq1ODwf40WdGVoE266UBUg==", + "dev": true, + "dependencies": { + "json-schema-ref-parser": "^6.1.0", + "jsonpath-plus": "^7.2.0" + }, + "bin": { + "jsf": "bin/gen.cjs" + } + }, + "node_modules/json-schema-ref-parser": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/json-schema-ref-parser/-/json-schema-ref-parser-6.1.0.tgz", + "integrity": "sha512-pXe9H1m6IgIpXmE5JSb8epilNTGsmTb2iPohAXpOdhqGFbQjNeHHsZxU+C8w6T81GZxSPFLeUoqDJmzxx5IGuw==", + "deprecated": "Please switch to @apidevtools/json-schema-ref-parser", + "dev": true, + "dependencies": { + "call-me-maybe": "^1.0.1", + "js-yaml": "^3.12.1", + "ono": "^4.0.11" + } + }, + "node_modules/json-schema-ref-parser/node_modules/argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "dev": true, + "dependencies": { + "sprintf-js": "~1.0.2" + } + }, + "node_modules/json-schema-ref-parser/node_modules/js-yaml": { + "version": "3.14.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", + "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", + "dev": true, + "dependencies": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, "node_modules/json-schema-traverse": { "version": "0.4.1", "dev": true, @@ -4555,6 +4648,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/jsonpath-plus": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/jsonpath-plus/-/jsonpath-plus-7.2.0.tgz", + "integrity": "sha512-zBfiUPM5nD0YZSBT/o/fbCUlCcepMIdP0CJZxM1+KgA4f2T206f6VAg9e7mX35+KlMaIc5qXW34f3BnwJ3w+RA==", + "dev": true, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/junk": { "version": "4.0.1", "dev": true, @@ -5744,6 +5846,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/ono": { + "version": "4.0.11", + "resolved": "https://registry.npmjs.org/ono/-/ono-4.0.11.tgz", + "integrity": "sha512-jQ31cORBFE6td25deYeD80wxKBMj+zBmHTrVxnc6CKhx8gho6ipmWM5zj/oeoqioZ99yqBls9Z/9Nss7J26G2g==", + "dev": true, + "dependencies": { + "format-util": "^1.0.3" + } + }, "node_modules/open": { "version": "7.4.2", "resolved": "https://registry.npmjs.org/open/-/open-7.4.2.tgz", @@ -6466,6 +6577,18 @@ "version": "1.0.0", "license": "MIT" }, + "node_modules/random-bytes-readable-stream": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/random-bytes-readable-stream/-/random-bytes-readable-stream-3.0.0.tgz", + "integrity": "sha512-GrDPlkikCTvAOClNgxbbZg2Xq84lmBqhCkkPuDh92vgsDfW9dvrp4kQpE7AOL/3B3j3BATkQocpnCp/bUBn9NQ==", + "dev": true, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/random-bytes-seed": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/random-bytes-seed/-/random-bytes-seed-1.0.3.tgz", @@ -7930,9 +8053,9 @@ } }, "node_modules/type-fest": { - "version": "4.5.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.5.0.tgz", - "integrity": "sha512-diLQivFzddJl4ylL3jxSkEc39Tpw7o1QeEHIPxVwryDK2lpB7Nqhzhuo6v5/Ls08Z0yPSAhsyAWlv1/H0ciNmw==", + "version": "4.8.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.8.0.tgz", + "integrity": "sha512-rIY1yHlQhXNRfRyUNnpBr9pr1qxCHSN80hNNHINWQvpgvrVnu3uoi20+mkRfSD1vud6fsA2VLU8AENZhj5jGCQ==", "engines": { "node": ">=16" }, diff --git a/package.json b/package.json index 9ca7c8675..cb76194da 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", + "@mapeo/mock-data": "github:digidem/mapeo-mock-data#feat/sync-generate", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", @@ -95,6 +96,7 @@ "prettier": "^2.8.8", "random-access-file": "^4.0.4", "random-access-memory": "^6.2.0", + "random-bytes-readable-stream": "^3.0.0", "rimraf": "^5.0.1", "streamx": "^2.15.1", "tempy": "^3.1.0", diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index ddcd42881..abc5227b8 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -10,7 +10,7 @@ import { Logger } from '../logger.js' */ /** @type {Namespace[]} */ -const PRESYNC_NAMESPACES = ['auth', 'config', 'blobIndex'] +export const PRESYNC_NAMESPACES = ['auth', 'config', 'blobIndex'] export class PeerSyncController { #replicatingCores = new Set() diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index e3b56c161..f2ebbe5bf 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -1,7 +1,11 @@ import { TypedEmitter } from 'tiny-typed-emitter' import { SyncState } from './sync-state.js' -import { PeerSyncController } from './peer-sync-controller.js' +import { + PeerSyncController, + PRESYNC_NAMESPACES, +} from './peer-sync-controller.js' import { Logger } from '../logger.js' +import { NAMESPACES } from '../core-manager/index.js' export const kHandleDiscoveryKey = Symbol('handle discovery key') @@ -103,6 +107,22 @@ export class SyncApi extends TypedEmitter { } } + /** + * @param {'initial' | 'full'} type + */ + async waitForSync(type) { + const state = this.getState() + const namespaces = type === 'initial' ? PRESYNC_NAMESPACES : NAMESPACES + if (isSynced(state, namespaces, this.#peerSyncControllers.size)) return + return new Promise((res) => { + this.on('sync-state', function onState(state) { + if (!isSynced(state, namespaces, this.#peerSyncControllers.size)) return + this.off('sync-state', onState) + res(null) + }) + }) + } + /** * Bound to `this` * @@ -165,3 +185,22 @@ export class SyncApi extends TypedEmitter { this.#pendingDiscoveryKeys.delete(protomux) } } + +/** + * Is the sync state "synced", e.g. is there nothing left to sync + * + * @param {import('./sync-state.js').State} state + * @param {readonly import('../core-manager/index.js').Namespace[]} namespaces + * @param {number} peerCount + */ +function isSynced(state, namespaces, peerCount) { + for (const ns of namespaces) { + if (state[ns].dataToSync) return false + const remoteStates = Object.values(state[ns].remoteStates) + if (remoteStates.length !== peerCount) return false + for (const rs of remoteStates) { + if (rs.status === 'connecting') return false + } + } + return true +} diff --git a/test-e2e/project-crud.js b/test-e2e/project-crud.js index 4d344fe19..a56ad9282 100644 --- a/test-e2e/project-crud.js +++ b/test-e2e/project-crud.js @@ -4,6 +4,8 @@ import { KeyManager } from '@mapeo/crypto' import { valueOf } from '../src/utils.js' import { MapeoManager } from '../src/mapeo-manager.js' import RAM from 'random-access-memory' +import { stripUndef } from './utils-new.js' +import { round } from './utils-new.js' /** @satisfies {Array} */ const fixtures = [ @@ -126,20 +128,3 @@ test('CRUD operations', async (t) => { }) } }) - -/** - * Remove undefined properties from an object, to allow deep comparison - * @param {object} obj - */ -function stripUndef(obj) { - return JSON.parse(JSON.stringify(obj)) -} - -/** - * - * @param {number} value - * @param {number} decimalPlaces - */ -function round(value, decimalPlaces) { - return Math.round(value * 10 ** decimalPlaces) / 10 ** decimalPlaces -} diff --git a/test-e2e/sync.js b/test-e2e/sync.js new file mode 100644 index 000000000..fb837f2ae --- /dev/null +++ b/test-e2e/sync.js @@ -0,0 +1,119 @@ +import { test } from 'brittle' +import { + connectPeers, + createManagers, + invite, + seedDatabases, + sortById, + waitForSync, +} from './utils-new.js' +import { kCoreManager } from '../src/mapeo-project.js' +import { getKeys } from '../tests/helpers/core-manager.js' +import { NAMESPACES } from '../src/core-manager/index.js' +import { PRESYNC_NAMESPACES } from '../src/sync/peer-sync-controller.js' + +const SCHEMAS_INITIAL_SYNC = ['preset', 'field'] + +test('Create and sync data', async function (t) { + const COUNT = 5 + const managers = await createManagers(COUNT) + const [invitor, ...invitees] = managers + const disconnect = connectPeers(managers, { discovery: false }) + const projectId = await invitor.createProject() + await invite({ invitor, invitees, projectId }) + await disconnect() + + const projects = await Promise.all( + managers.map((m) => m.getProject(projectId)) + ) + + const generatedDocs = (await seedDatabases(projects)).flat() + t.pass(`Generated ${generatedDocs.length} values`) + const generatedSchemaNames = generatedDocs.reduce((acc, cur) => { + acc.add(cur.schemaName) + return acc + }, new Set()) + + connectPeers(managers, { discovery: false }) + await waitForSync(projects, 'initial') + + for (const schemaName of generatedSchemaNames) { + for (const project of projects) { + const deviceId = project.deviceId.slice(0, 7) + // @ts-ignore - to complex to narrow `schemaName` to valid values + const docs = await project[schemaName].getMany() + const expected = generatedDocs.filter((v) => v.schemaName === schemaName) + if (SCHEMAS_INITIAL_SYNC.includes(schemaName)) { + t.alike( + sortById(docs), + sortById(expected), + `All ${schemaName} docs synced to ${deviceId}` + ) + } else { + t.not( + docs.length, + expected.length, + `Not all ${schemaName} docs synced to ${deviceId}` + ) + } + } + } + + for (const project of projects) { + project.$sync.start() + } + + await waitForSync(projects, 'full') + + for (const schemaName of generatedSchemaNames) { + for (const project of projects) { + const deviceId = project.deviceId.slice(0, 7) + // @ts-ignore - to complex to narrow `schemaName` to valid values + const docs = await project[schemaName].getMany() + const expected = generatedDocs.filter((v) => v.schemaName === schemaName) + t.alike( + sortById(docs), + sortById(expected), + `All ${schemaName} docs synced to ${deviceId}` + ) + } + } +}) + +test('shares cores', async function (t) { + const COUNT = 5 + const managers = await createManagers(COUNT) + const [invitor, ...invitees] = managers + connectPeers(managers, { discovery: false }) + const projectId = await invitor.createProject() + await invite({ invitor, invitees, projectId }) + + const projects = await Promise.all( + managers.map((m) => m.getProject(projectId)) + ) + const coreManagers = projects.map((p) => p[kCoreManager]) + + await waitForSync(projects, 'initial') + + for (const ns of PRESYNC_NAMESPACES) { + for (const cm of coreManagers) { + const keyCount = getKeys(cm, ns).length + t.is(keyCount, COUNT, 'expected number of cores') + } + } + + // Currently need to start syncing to share other keys - this might change if + // we add keys based on coreOwnership records + for (const project of projects) { + project.$sync.start() + } + + await waitForSync(projects, 'full') + + for (const ns of NAMESPACES) { + for (const cm of coreManagers) { + const keyCount = getKeys(cm, ns).length + t.is(keyCount, COUNT, 'expected number of cores') + } + } +}) diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js index 37e8aee4c..8e8ca061d 100644 --- a/test-e2e/utils-new.js +++ b/test-e2e/utils-new.js @@ -3,9 +3,13 @@ import sodium from 'sodium-universal' import RAM from 'random-access-memory' import { MapeoManager } from '../src/index.js' -import { kRPC } from '../src/mapeo-manager.js' +import { kManagerReplicate, kRPC } from '../src/mapeo-manager.js' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { once } from 'node:events' +// @ts-expect-error - pending publishing module with types +import { generate } from '@mapeo/mock-data' +import { valueOf } from '../src/utils.js' +import { randomInt } from 'node:crypto' /** * @param {readonly MapeoManager[]} managers @@ -26,8 +30,36 @@ export function connectPeers(managers, { discovery = true } = {}) { for (const manager of managers) { manager.startLocalPeerDiscovery() } + return function destroy() { + return disconnectPeers(managers) + } } else { - // TODO: replicate all managers, for faster tests (discovery can take extra time) + /** @type {import('../src/types.js').ReplicationStream[]} */ + const replicationStreams = [] + for (let i = 0; i < managers.length; i++) { + for (let j = i + 1; j < managers.length; j++) { + const r1 = managers[i][kManagerReplicate](true) + const r2 = managers[j][kManagerReplicate](false) + replicationStreams.push(r1, r2) + // @ts-ignore - either the types or wrong, or we're returning the wrong thing + r1.rawStream.pipe(r2.rawStream).pipe(r1.rawStream) + } + } + return function destroy() { + const promises = [] + for (const stream of replicationStreams) { + promises.push( + /** @type {Promise} */ + ( + new Promise((res) => { + stream.on('close', res) + stream.destroy() + }) + ) + ) + } + return Promise.all(promises) + } } } @@ -145,3 +177,109 @@ function getRootKey(seed) { } return key } +/** + * Remove undefined properties from an object, to allow deep comparison + * @param {object} obj + */ +export function stripUndef(obj) { + return JSON.parse(JSON.stringify(obj)) +} +/** + * + * @param {number} value + * @param {number} decimalPlaces + */ +export function round(value, decimalPlaces) { + return Math.round(value * 10 ** decimalPlaces) / 10 ** decimalPlaces +} + +/** + * Unlike `mapeo.project.$sync.waitForSync` this also waits for the specified + * number of peers to connect. + * + * @param {import('../src/mapeo-project.js').MapeoProject} project + * @param {number} peerCount + * @param {'initial' | 'full'} [type] + */ +async function waitForProjectSync(project, peerCount, type = 'initial') { + const state = await project.$sync.getState() + if (Object.keys(state.auth.remoteStates).length === peerCount) { + return project.$sync.waitForSync(type) + } + return new Promise((res) => { + project.$sync.on('sync-state', function onState(state) { + if (Object.keys(state.auth.remoteStates).length !== peerCount) return + project.$sync.off('sync-state', onState) + res(project.$sync.waitForSync(type)) + }) + }) +} + +/** + * Wait for all projects to connect and sync + * + * @param {import('../src/mapeo-project.js').MapeoProject[]} projects + * @param {'initial' | 'full'} [type] + */ +export function waitForSync(projects, type = 'initial') { + return Promise.all( + projects.map((project) => { + return waitForProjectSync(project, projects.length - 1, type) + }) + ) +} + +/** + * @param {import('../src/mapeo-project.js').MapeoProject[]} projects + */ +export function seedDatabases(projects) { + return Promise.all(projects.map((p) => seedProjectDatabase(p))) +} + +const SCHEMAS_TO_SEED = /** @type {const} */ ([ + 'observation', + 'preset', + 'field', +]) + +/** + * @param {import('../src/mapeo-project.js').MapeoProject} project + * @returns {Promise>} + */ +async function seedProjectDatabase(project) { + const promises = [] + for (const schemaName of SCHEMAS_TO_SEED) { + const count = + schemaName === 'observation' ? randomInt(20, 100) : randomInt(0, 10) + let i = 0 + while (i++ < count) { + const value = valueOf( + // @ts-ignore + generate(schemaName)[0] + ) + promises.push( + // @ts-ignore + project[schemaName].create(value) + ) + } + } + return Promise.all(promises) +} + +/** + * @template {object} T + * @param {T[]} arr + * @param {keyof T} key + */ +export function sortBy(arr, key) { + return arr.sort(function (a, b) { + if (a[key] < b[key]) return -1 + if (a[key] > b[key]) return 1 + return 0 + }) +} + +/** @param {import('@mapeo/schema').MapeoDoc[]} docs */ +export function sortById(docs) { + return sortBy(docs, 'docId') +} From bbfe301eeb7870a5ec7b8916426872e2b00f8981 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 12:31:33 +0900 Subject: [PATCH 52/69] add published @mapeo/mock-data --- package-lock.json | 8 ++++---- package.json | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index e51054265..b2f4ca324 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", - "@mapeo/mock-data": "github:digidem/mapeo-mock-data#feat/sync-generate", + "@mapeo/mock-data": "^1.0.0", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", @@ -851,10 +851,10 @@ } }, "node_modules/@mapeo/mock-data": { - "version": "0.0.0", - "resolved": "git+ssh://git@github.com/digidem/mapeo-mock-data.git#6cc68aa1acc6f4b7482a758df7ab00984fa7707d", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mapeo/mock-data/-/mock-data-1.0.0.tgz", + "integrity": "sha512-C2Vzu7QCbXawPAZrTVxTd7Ht2amBtA8CUMJOF64YdRNxXHInt0XRZAu7P1VH8joOQv6Ffn7T+n9U0pgke3pv8A==", "dev": true, - "license": "MIT", "dependencies": { "@faker-js/faker": "^8.3.1", "@mapeo/schema": "^3.0.0-next.11", diff --git a/package.json b/package.json index cb76194da..5fe1adc3b 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", - "@mapeo/mock-data": "github:digidem/mapeo-mock-data#feat/sync-generate", + "@mapeo/mock-data": "^1.0.0", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", From ea442a35ef7113dd98dcb3bd984739f333e314f3 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 12:50:36 +0900 Subject: [PATCH 53/69] fix: don't open cores in sparse mode Turns out this changes how core.length etc. work, which confuses things --- src/core-manager/index.js | 5 +++-- types/corestore.d.ts | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index 4f52a0c2f..7e0ca3a5f 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -285,9 +285,10 @@ export class CoreManager extends TypedEmitter { const core = this.#corestore.get({ keyPair, encryptionKey: this.#encryptionKeys[namespace], - // Starts live download of core immediately - sparse: namespace === 'blob', }) + if (namespace !== 'blob') { + core.download({ start: 0, end: -1 }) + } // Every peer adds a listener, so could have many peers core.setMaxListeners(0) // @ts-ignore - ensure key is defined before hypercore is ready diff --git a/types/corestore.d.ts b/types/corestore.d.ts index d0518074d..41545c146 100644 --- a/types/corestore.d.ts +++ b/types/corestore.d.ts @@ -27,7 +27,7 @@ declare module 'corestore' { options: Omit & { key?: Buffer | string | undefined keyPair: { publicKey: Buffer; secretKey?: Buffer | undefined | null } - sparse: boolean + sparse?: boolean } ): Hypercore replicate: typeof Hypercore.prototype.replicate From 72f0dff1c86990ca4a7a754c627b7716d0fabcbf Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 13:33:51 +0900 Subject: [PATCH 54/69] fix: option to skip auto download for tests --- src/core-manager/index.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index 7e0ca3a5f..ed1e094d4 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -59,6 +59,7 @@ export class CoreManager extends TypedEmitter { * TODO: Remove items from this set after a max age */ #keyRequests = new TrackedKeyRequests() + #autoDownload static get namespaces() { return NAMESPACES @@ -72,6 +73,7 @@ export class CoreManager extends TypedEmitter { * @param {Buffer} [options.projectSecretKey] 32-byte secret key of the project creator core * @param {Partial>} [options.encryptionKeys] Encryption keys for each namespace * @param {import('hypercore').HypercoreStorage} options.storage Folder to store all hypercore data + * @param {boolean} [options.autoDownload=true] Immediately start downloading cores - should only be set to false for tests * @param {Logger} [options.logger] */ constructor({ @@ -81,6 +83,7 @@ export class CoreManager extends TypedEmitter { projectSecretKey, encryptionKeys = {}, storage, + autoDownload = true, logger, }) { super() @@ -99,6 +102,7 @@ export class CoreManager extends TypedEmitter { this.#deviceId = keyManager.getIdentityKeypair().publicKey.toString('hex') this.#projectKey = projectKey this.#encryptionKeys = encryptionKeys + this.#autoDownload = autoDownload // Make sure table exists for persisting known cores sqlite.prepare(CREATE_SQL).run() @@ -286,7 +290,7 @@ export class CoreManager extends TypedEmitter { keyPair, encryptionKey: this.#encryptionKeys[namespace], }) - if (namespace !== 'blob') { + if (namespace !== 'blob' && this.#autoDownload) { core.download({ start: 0, end: -1 }) } // Every peer adds a listener, so could have many peers From a07c278c75c3cf6aecb3c3475533bdf963ea122e Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 15:27:28 +0900 Subject: [PATCH 55/69] e2e test for stop-start sync --- package-lock.json | 8 ++--- package.json | 2 +- src/utils.js | 2 +- test-e2e/sync.js | 71 +++++++++++++++++++++++++++++++++++++++++++ test-e2e/utils-new.js | 6 +--- 5 files changed, 78 insertions(+), 11 deletions(-) diff --git a/package-lock.json b/package-lock.json index b2f4ca324..d4b449f98 100644 --- a/package-lock.json +++ b/package-lock.json @@ -55,7 +55,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", - "@mapeo/mock-data": "^1.0.0", + "@mapeo/mock-data": "^1.0.1", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", @@ -851,9 +851,9 @@ } }, "node_modules/@mapeo/mock-data": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@mapeo/mock-data/-/mock-data-1.0.0.tgz", - "integrity": "sha512-C2Vzu7QCbXawPAZrTVxTd7Ht2amBtA8CUMJOF64YdRNxXHInt0XRZAu7P1VH8joOQv6Ffn7T+n9U0pgke3pv8A==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@mapeo/mock-data/-/mock-data-1.0.1.tgz", + "integrity": "sha512-Mdmyn3dCjF3wuwuGJhjGjVnydwHoZPgpAyA8JJvZDBNr1qUbeQyq6mrBr+Tg90ZBRY7G9S0wxABD+OJOKZhQdg==", "dev": true, "dependencies": { "@faker-js/faker": "^8.3.1", diff --git a/package.json b/package.json index 5fe1adc3b..ad3873029 100644 --- a/package.json +++ b/package.json @@ -69,7 +69,7 @@ "devDependencies": { "@bufbuild/buf": "^1.26.1", "@hyperswarm/testnet": "^3.1.2", - "@mapeo/mock-data": "^1.0.0", + "@mapeo/mock-data": "^1.0.1", "@sinonjs/fake-timers": "^10.0.2", "@types/b4a": "^1.6.0", "@types/debug": "^4.1.8", diff --git a/src/utils.js b/src/utils.js index 07fc064ea..1da1b0d5d 100644 --- a/src/utils.js +++ b/src/utils.js @@ -85,7 +85,7 @@ export function deNullify(obj) { } /** - * @template {import('@mapeo/schema').MapeoDoc & { forks: string[] }} T + * @template {import('@mapeo/schema').MapeoDoc & { forks?: string[] }} T * @param {T} doc * @returns {Omit} */ diff --git a/test-e2e/sync.js b/test-e2e/sync.js index fb837f2ae..d26f374d5 100644 --- a/test-e2e/sync.js +++ b/test-e2e/sync.js @@ -11,6 +11,9 @@ import { kCoreManager } from '../src/mapeo-project.js' import { getKeys } from '../tests/helpers/core-manager.js' import { NAMESPACES } from '../src/core-manager/index.js' import { PRESYNC_NAMESPACES } from '../src/sync/peer-sync-controller.js' +import { generate } from '@mapeo/mock-data' +import { valueOf } from '../src/utils.js' +import pTimeout from 'p-timeout' const SCHEMAS_INITIAL_SYNC = ['preset', 'field'] @@ -80,6 +83,74 @@ test('Create and sync data', async function (t) { } }) +test('start and stop sync', async function (t) { + // Checks that both peers need to start syncing for data to sync, and that + // $sync.stop() actually stops data syncing + const COUNT = 2 + const managers = await createManagers(COUNT) + const [invitor, ...invitees] = managers + const disconnect = connectPeers(managers, { discovery: false }) + const projectId = await invitor.createProject() + await invite({ invitor, invitees, projectId }) + + const projects = await Promise.all( + managers.map((m) => m.getProject(projectId)) + ) + const [invitorProject, inviteeProject] = projects + + const obs1 = await invitorProject.observation.create( + valueOf(generate('observation')[0]) + ) + await waitForSync(projects, 'initial') + inviteeProject.$sync.start() + + await t.exception( + () => pTimeout(waitForSync(projects, 'full'), { milliseconds: 1000 }), + 'wait for sync times out' + ) + + await t.exception( + () => inviteeProject.observation.getByDocId(obs1.docId), + 'before both peers have started sync, doc does not sync' + ) + + invitorProject.$sync.start() + + // Use the same timeout as above, to check that it would have synced given the timeout + await pTimeout(waitForSync(projects, 'full'), { milliseconds: 1000 }) + + const obs1Synced = await inviteeProject.observation.getByDocId(obs1.docId) + + t.alike(obs1Synced, obs1, 'observation is synced') + + inviteeProject.$sync.stop() + + const obs2 = await inviteeProject.observation.create( + valueOf(generate('observation')[0]) + ) + await waitForSync(projects, 'initial') + + await t.exception( + () => pTimeout(waitForSync(projects, 'full'), { milliseconds: 1000 }), + 'wait for sync times out' + ) + + await t.exception( + () => invitorProject.observation.getByDocId(obs2.docId), + 'after stopping sync, data does not sync' + ) + + inviteeProject.$sync.start() + + await pTimeout(waitForSync(projects, 'full'), { milliseconds: 1000 }) + + const obs2Synced = await invitorProject.observation.getByDocId(obs2.docId) + + t.alike(obs2Synced, obs2, 'observation is synced') + + await disconnect() +}) + test('shares cores', async function (t) { const COUNT = 5 const managers = await createManagers(COUNT) diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js index 8e8ca061d..7635549e1 100644 --- a/test-e2e/utils-new.js +++ b/test-e2e/utils-new.js @@ -6,7 +6,6 @@ import { MapeoManager } from '../src/index.js' import { kManagerReplicate, kRPC } from '../src/mapeo-manager.js' import { MEMBER_ROLE_ID } from '../src/capabilities.js' import { once } from 'node:events' -// @ts-expect-error - pending publishing module with types import { generate } from '@mapeo/mock-data' import { valueOf } from '../src/utils.js' import { randomInt } from 'node:crypto' @@ -253,10 +252,7 @@ async function seedProjectDatabase(project) { schemaName === 'observation' ? randomInt(20, 100) : randomInt(0, 10) let i = 0 while (i++ < count) { - const value = valueOf( - // @ts-ignore - generate(schemaName)[0] - ) + const value = valueOf(generate(schemaName)[0]) promises.push( // @ts-ignore project[schemaName].create(value) From 800d1364375a6d27535252513ac88734bcc21bbb Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 15:29:16 +0900 Subject: [PATCH 56/69] fix coreManager unit tests --- src/core-manager/index.js | 56 ++++++++ src/mapeo-project.js | 8 ++ tests/core-manager.js | 235 ++++------------------------------ tests/helpers/core-manager.js | 12 +- 4 files changed, 96 insertions(+), 215 deletions(-) diff --git a/src/core-manager/index.js b/src/core-manager/index.js index ed1e094d4..9168449a1 100644 --- a/src/core-manager/index.js +++ b/src/core-manager/index.js @@ -8,7 +8,9 @@ import * as rle from './bitfield-rle.js' import { Logger } from '../logger.js' import { keyToId } from '../utils.js' import { discoveryKey } from 'hypercore-crypto' +import Hypercore from 'hypercore' +export const kCoreManagerReplicate = Symbol('replicate core manager') // WARNING: Changing these will break things for existing apps, since namespaces // are used for key derivation export const NAMESPACES = /** @type {const} */ ([ @@ -454,6 +456,28 @@ export class CoreManager extends TypedEmitter { peer.protomux.uncork() } + + /** + * ONLY FOR TESTING + * Replicate all cores in core manager + * + * @param {Parameters[0]} stream + * @returns + */ + [kCoreManagerReplicate](stream) { + const protocolStream = Hypercore.createProtocolStream(stream, { + ondiscoverykey: async (discoveryKey) => { + const peer = await findPeer( + this.creatorCore, + // @ts-ignore + protocolStream.noiseStream.remotePublicKey + ) + if (!peer) return + this.requestCoreKey(peer.remotePublicKey, discoveryKey) + }, + }) + return this.#corestore.replicate(stream) + } } /** @@ -557,3 +581,35 @@ class TrackedKeyRequests { this.#byPeerId.clear() } } + +/** + * @param {Hypercore<"binary", Buffer>} core + * @param {Buffer} publicKey + * @param {{ timeout?: number }} [opts] + */ +function findPeer(core, publicKey, { timeout = 200 } = {}) { + const peer = core.peers.find((peer) => { + return peer.remotePublicKey.equals(publicKey) + }) + if (peer) return peer + // This is called from the from the handleDiscoveryId event, which can + // happen before the peer connection is fully established, so we wait for + // the `peer-add` event, with a timeout in case the peer never gets added + return new Promise(function (res) { + const timeoutId = setTimeout(function () { + core.off('peer-add', onPeer) + res(null) + }, timeout) + + core.on('peer-add', onPeer) + + /** @param {any} peer */ + function onPeer(peer) { + if (peer.remotePublicKey.equals(publicKey)) { + clearTimeout(timeoutId) + core.off('peer-add', onPeer) + res(peer) + } + } + }) +} diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 43bc042a0..9c759e857 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -44,6 +44,7 @@ import { IconApi } from './icon-api.js' const CORESTORE_STORAGE_FOLDER_NAME = 'corestore' const INDEXER_STORAGE_FOLDER_NAME = 'indexer' +export const kCoreManager = Symbol('coreManager') export const kCoreOwnership = Symbol('coreOwnership') export const kCapabilities = Symbol('capabilities') export const kSetOwnDeviceInfo = Symbol('kSetOwnDeviceInfo') @@ -325,6 +326,13 @@ export class MapeoProject { this.#l.log('Created project instance %h', projectKey) } + /** + * CoreManager instance, used for tests + */ + get [kCoreManager]() { + return this.#coreManager + } + /** * CoreOwnership instance, used for tests */ diff --git a/tests/core-manager.js b/tests/core-manager.js index cb7e2b2cf..5be685fcd 100644 --- a/tests/core-manager.js +++ b/tests/core-manager.js @@ -6,7 +6,10 @@ import { createCoreManager, replicate } from './helpers/core-manager.js' import { randomBytes } from 'crypto' import Sqlite from 'better-sqlite3' import { KeyManager } from '@mapeo/crypto' -import { CoreManager, unreplicate } from '../src/core-manager/index.js' +import { + CoreManager, + kCoreManagerReplicate, +} from '../src/core-manager/index.js' import RemoteBitfield from '../src/core-manager/remote-bitfield.js' import assert from 'assert' import { once } from 'node:events' @@ -15,8 +18,8 @@ import { exec } from 'child_process' import { RandomAccessFilePool } from '../src/core-manager/random-access-file-pool.js' import RandomAccessFile from 'random-access-file' import path from 'path' -import { setTimeout as delay } from 'node:timers/promises' import { Transform } from 'streamx' +import { waitForCores } from './helpers/core-manager.js' async function createCore(key) { const core = new Hypercore(RAM, key) @@ -24,76 +27,6 @@ async function createCore(key) { return core } -test('shares auth cores', async function (t) { - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - - replicate(cm1, cm2) - - await Promise.all([ - waitForCores(cm1, getKeys(cm2, 'auth')), - waitForCores(cm2, getKeys(cm1, 'auth')), - ]) - - const cm1Keys = getKeys(cm1, 'auth').sort(Buffer.compare) - const cm2Keys = getKeys(cm2, 'auth').sort(Buffer.compare) - - t.alike(cm1Keys, cm2Keys, 'Share same auth cores') -}) - -test('shares other cores', async function (t) { - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - - const { - rsm: [rsm1, rsm2], - } = replicate(cm1, cm2) - - for (const namespace of ['config', 'data', 'blob', 'blobIndex']) { - rsm1.enableNamespace(namespace) - rsm2.enableNamespace(namespace) - await Promise.all([ - waitForCores(cm1, getKeys(cm2, namespace)), - waitForCores(cm2, getKeys(cm1, namespace)), - ]) - const cm1Keys = getKeys(cm1, namespace).sort(Buffer.compare) - const cm2Keys = getKeys(cm2, namespace).sort(Buffer.compare) - - t.alike(cm1Keys, cm2Keys, `Share same ${namespace} cores`) - } -}) - -// Testing this case because in real-use namespaces are not enabled at the same time -test('shares cores if namespaces enabled at different times', async function (t) { - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - - const { - rsm: [rsm1, rsm2], - } = replicate(cm1, cm2) - - for (const namespace of ['config', 'data', 'blob', 'blobIndex']) { - rsm1.enableNamespace(namespace) - } - - await delay(1000) - - for (const namespace of ['config', 'data', 'blob', 'blobIndex']) { - rsm2.enableNamespace(namespace) - await Promise.all([ - waitForCores(cm1, getKeys(cm2, namespace)), - waitForCores(cm2, getKeys(cm1, namespace)), - ]) - const cm1Keys = getKeys(cm1, namespace).sort(Buffer.compare) - const cm2Keys = getKeys(cm2, namespace).sort(Buffer.compare) - - t.alike(cm1Keys, cm2Keys, `Share same ${namespace} cores`) - } -}) - test('project creator auth core has project key', async function (t) { const sqlite = new Sqlite(':memory:') const keyManager = new KeyManager(randomBytes(16)) @@ -167,7 +100,6 @@ test('eagerly updates remote bitfields', async function (t) { .done() await destroyReplication() await cm1Core.clear(0, 2) - { // This is ensuring that bitfields also get propogated in the other // direction, e.g. from the non-writer to the writer @@ -224,62 +156,6 @@ test('eagerly updates remote bitfields', async function (t) { } }) -test('works with an existing protocol stream for replications', async function (t) { - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - - const n1 = new NoiseSecretStream(true) - const n2 = new NoiseSecretStream(false) - n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream) - - const s1 = Hypercore.createProtocolStream(n1) - const s2 = Hypercore.createProtocolStream(n2) - - cm1.replicate(s1) - cm2.replicate(s2) - - await Promise.all([ - waitForCores(cm1, getKeys(cm2, 'auth')), - waitForCores(cm2, getKeys(cm1, 'auth')), - ]) - - const cm1Keys = getKeys(cm1, 'auth').sort(Buffer.compare) - const cm2Keys = getKeys(cm2, 'auth').sort(Buffer.compare) - - t.alike(cm1Keys, cm2Keys, 'Share same auth cores') -}) - -test.skip('can mux other project replications over same stream', async function (t) { - // This test fails because https://github.com/holepunchto/corestore/issues/45 - // The `ondiscoverykey` hook for `Hypercore.createProtocolStream()` that we - // use to know when other cores are muxed in the stream is only called the - // first time the protocol stream is created. When a second core replicates - // to the same stream, it sees it is already a protomux stream, and it does - // not add the notify hook for `ondiscoverykey`. - // We might be able to work around this if we want to enable multi-project - // muxing before the issue is resolved by creating the protomux stream outside - // the core manager, and then somehow hooking into the relevant corestore. - t.plan(2) - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - const otherProject = createCoreManager() - - const n1 = new NoiseSecretStream(true) - const n2 = new NoiseSecretStream(false) - n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream) - - await Promise.all([ - waitForCores(cm1, getKeys(cm2, 'auth')), - waitForCores(cm2, getKeys(cm1, 'auth')), - ]) - - cm1.replicate(n1) - otherProject.replicate(n2) - cm2.replicate(n2) -}) - test('multiplexing waits for cores to be added', async function (t) { // Mapeo code expects replication to work when cores are not added to the // replication stream at the same time. This is not explicitly tested in @@ -326,8 +202,6 @@ test('close()', async (t) => { t.is(core.sessions.length, 0, 'no open sessions') } } - const ns = new NoiseSecretStream(true) - t.exception(() => cm.replicate(ns), /closed/) }) test('Added cores are persisted', async (t) => { @@ -365,14 +239,8 @@ test('encryption', async function (t) { const cm2 = createCoreManager({ projectKey }) const cm3 = createCoreManager({ projectKey, encryptionKeys }) - const { rsm: rsm1 } = replicate(cm1, cm2) - const { rsm: rsm2 } = replicate(cm1, cm3) - - for (const rsm of [...rsm1, ...rsm2]) { - for (const ns of CoreManager.namespaces) { - rsm.enableNamespace(ns) - } - } + replicate(cm1, cm2) + replicate(cm1, cm3) for (const ns of CoreManager.namespaces) { const { core, key } = cm1.getWriterCore(ns) @@ -442,39 +310,6 @@ test('poolSize limits number of open file descriptors', async function (t) { }) }) -async function waitForCores(coreManager, keys) { - const allKeys = getAllKeys(coreManager) - if (hasKeys(keys, allKeys)) return - return new Promise((res) => { - coreManager.on('add-core', function onAddCore({ key }) { - allKeys.push(key) - if (hasKeys(keys, allKeys)) { - coreManager.off('add-core', onAddCore) - res() - } - }) - }) -} - -function getAllKeys(coreManager) { - const keys = [] - for (const namespace of CoreManager.namespaces) { - keys.push.apply(keys, getKeys(coreManager, namespace)) - } - return keys -} - -function getKeys(coreManager, namespace) { - return coreManager.getCores(namespace).map(({ key }) => key) -} - -function hasKeys(someKeys, allKeys) { - for (const key of someKeys) { - if (!allKeys.find((k) => k.equals(key))) return false - } - return true -} - test('sends "haves" bitfields over project creator core replication stream', async function (t) { const projectKey = randomBytes(32) const cm1 = createCoreManager({ projectKey }) @@ -510,8 +345,8 @@ test('sends "haves" bitfields over project creator core replication stream', asy const cm1Core = cm1.getWriterCore('data').core await cm1Core.ready() const batchSize = 4096 - // Create 1 million entries in hypercore - for (let i = 0; i < 2 ** 20; i += batchSize) { + // Create 4 million entries in hypercore - will be at least two have bitfields + for (let i = 0; i < 2 ** 22; i += batchSize) { const data = Array(batchSize) .fill(null) .map(() => 'block') @@ -525,8 +360,8 @@ test('sends "haves" bitfields over project creator core replication stream', asy const n2 = new NoiseSecretStream(false) n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream) - cm1.replicate(n1) - cm2.replicate(n2) + cm1[kCoreManagerReplicate](n1) + cm2[kCoreManagerReplicate](n2) // Need to wait for now, since no event for when a remote bitfield is updated await new Promise((res) => setTimeout(res, 200)) @@ -622,40 +457,6 @@ test('unreplicate', async (t) => { }) }) -test('disableNamespace and re-enable', async (t) => { - const projectKey = randomBytes(32) - const cm1 = createCoreManager({ projectKey }) - const cm2 = createCoreManager({ projectKey }) - - const { - rsm: [rsm1, rsm2], - } = replicate(cm1, cm2) - - rsm1.enableNamespace('data') - rsm2.enableNamespace('data') - - await Promise.all([ - waitForCores(cm1, getKeys(cm2, 'data')), - waitForCores(cm2, getKeys(cm1, 'data')), - ]) - - const data1CR = cm1.getWriterCore('data') - await data1CR.core.append(['a', 'b', 'c']) - - const data1ReplicaCore = cm2.getCoreByKey(data1CR.key) - t.is((await data1ReplicaCore.get(2, { timeout: 200 })).toString(), 'c') - - rsm1.disableNamespace('data') - - await data1CR.core.append(['d', 'e', 'f']) - - await t.exception(() => data1ReplicaCore.get(5, { timeout: 200 })) - - rsm1.enableNamespace('data') - - t.is((await data1ReplicaCore.get(5, { timeout: 200 })).toString(), 'f') -}) - const DEBUG = process.env.DEBUG // Compare two bitfields (instance of core.core.bitfield or peer.remoteBitfield) @@ -728,3 +529,17 @@ function latencyStream(delay = 0) { }, }) } + +/** + * + * @param {Hypercore<'binary', any>} core + * @param {import('protomux')} protomux + */ +export function unreplicate(core, protomux) { + const peerToUnreplicate = core.peers.find( + (peer) => peer.protomux === protomux + ) + if (!peerToUnreplicate) return + peerToUnreplicate.channel.close() + return +} diff --git a/tests/helpers/core-manager.js b/tests/helpers/core-manager.js index d7b2a5261..a4663fab4 100644 --- a/tests/helpers/core-manager.js +++ b/tests/helpers/core-manager.js @@ -1,11 +1,13 @@ // @ts-nocheck -import { CoreManager } from '../../src/core-manager/index.js' +import { + CoreManager, + kCoreManagerReplicate, +} from '../../src/core-manager/index.js' import Sqlite from 'better-sqlite3' import { randomBytes } from 'crypto' import { KeyManager } from '@mapeo/crypto' import RAM from 'random-access-memory' import NoiseSecretStream from '@hyperswarm/secret-stream' - /** * * @param {Partial[0]> & { rootKey?: Buffer }} param0 @@ -23,6 +25,7 @@ export function createCoreManager({ keyManager, storage: RAM, projectKey, + autoDownload: false, ...opts, }) } @@ -43,8 +46,8 @@ export function replicate( const n2 = new NoiseSecretStream(false, undefined, { keyPair: kp2 }) n1.rawStream.pipe(n2.rawStream).pipe(n1.rawStream) - const rsm1 = cm1.replicate(n1) - const rsm2 = cm2.replicate(n2) + cm1[kCoreManagerReplicate](n1) + cm2[kCoreManagerReplicate](n2) async function destroy() { await Promise.all([ @@ -60,7 +63,6 @@ export function replicate( } return { - rsm: [rsm1, rsm2], destroy, } } From 3afafbbbb86ce6a9a9bc0af56e50278c9dad97e1 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 15:57:08 +0900 Subject: [PATCH 57/69] fix blob store tests --- tests/blob-store/blob-store.js | 22 +++++++++++++--------- tests/helpers/blob-store.js | 21 --------------------- 2 files changed, 13 insertions(+), 30 deletions(-) diff --git a/tests/blob-store/blob-store.js b/tests/blob-store/blob-store.js index 254e73a48..c15cd6fc8 100644 --- a/tests/blob-store/blob-store.js +++ b/tests/blob-store/blob-store.js @@ -5,10 +5,14 @@ import { pipelinePromise as pipeline } from 'streamx' import { randomBytes } from 'node:crypto' import fs from 'fs' import { readFile } from 'fs/promises' -import { createCoreManager, waitForCores } from '../helpers/core-manager.js' +import { + replicate, + createCoreManager, + waitForCores, +} from '../helpers/core-manager.js' import { BlobStore } from '../../src/blob-store/index.js' import { setTimeout } from 'node:timers/promises' -import { replicateBlobs, concat } from '../helpers/blob-store.js' +import { concat } from '../helpers/blob-store.js' import { discoveryKey } from 'hypercore-crypto' // Test with buffers that are 3 times the default blockSize for hyperblobs @@ -86,7 +90,7 @@ test('get(), initialized but unreplicated drive', async (t) => { }) const driveId = await bs1.put(blob1Id, blob1) - const { destroy } = replicateBlobs(cm1, cm2) + const { destroy } = replicate(cm1, cm2) await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) /** @type {any} */ @@ -113,7 +117,7 @@ test('get(), replicated blobIndex, but blobs not replicated', async (t) => { }) const driveId = await bs1.put(blob1Id, blob1) - const { destroy } = replicateBlobs(cm1, cm2) + const { destroy } = replicate(cm1, cm2) await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) /** @type {any} */ const { core: replicatedCore } = cm2.getCoreByDiscoveryKey( @@ -277,13 +281,13 @@ test('live download', async function (t) { // STEP 1: Write a blob to CM1 const driveId1 = await bs1.put(blob1Id, blob1) // STEP 2: Replicate CM1 with CM3 - const { destroy: destroy1 } = replicateBlobs(cm1, cm3) + const { destroy: destroy1 } = replicate(cm1, cm3) // STEP 3: Start live download to CM3 const liveDownload = bs3.download() // STEP 4: Wait for blobs to be downloaded await downloaded(liveDownload) // STEP 5: Replicate CM2 with CM3 - const { destroy: destroy2 } = replicateBlobs(cm2, cm3) + const { destroy: destroy2 } = replicate(cm2, cm3) // STEP 6: Write a blob to CM2 const driveId2 = await bs2.put(blob2Id, blob2) // STEP 7: Wait for blobs to be downloaded @@ -332,7 +336,7 @@ test('sparse live download', async function (t) { await bs1.put(blob2Id, blob2) await bs1.put(blob3Id, blob3) - const { destroy } = replicateBlobs(cm1, cm2) + const { destroy } = replicate(cm1, cm2) const liveDownload = bs2.download({ photo: ['original', 'preview'] }) await downloaded(liveDownload) @@ -369,7 +373,7 @@ test('cancelled live download', async function (t) { // STEP 1: Write a blob to CM1 const driveId1 = await bs1.put(blob1Id, blob1) // STEP 2: Replicate CM1 with CM3 - const { destroy: destroy1 } = replicateBlobs(cm1, cm3) + const { destroy: destroy1 } = replicate(cm1, cm3) // STEP 3: Start live download to CM3 const ac = new AbortController() const liveDownload = bs3.download(undefined, { signal: ac.signal }) @@ -378,7 +382,7 @@ test('cancelled live download', async function (t) { // STEP 5: Cancel download ac.abort() // STEP 6: Replicate CM2 with CM3 - const { destroy: destroy2 } = replicateBlobs(cm2, cm3) + const { destroy: destroy2 } = replicate(cm2, cm3) // STEP 7: Write a blob to CM2 const driveId2 = await bs2.put(blob2Id, blob2) // STEP 8: Wait for blobs to (not) download diff --git a/tests/helpers/blob-store.js b/tests/helpers/blob-store.js index 2f49dd305..a56b248c4 100644 --- a/tests/helpers/blob-store.js +++ b/tests/helpers/blob-store.js @@ -1,5 +1,4 @@ // @ts-nocheck -import { replicate } from './core-manager.js' import { pipelinePromise as pipeline, Writable } from 'streamx' import { BlobStore } from '../../src/blob-store/index.js' @@ -16,26 +15,6 @@ export function createBlobStore(options = {}) { return { blobStore, coreManager } } -/** - * - * @param {import('../../src/core-manager/index.js').CoreManager} cm1 - * @param {import('../../src/core-manager/index.js').CoreManager} cm2 - */ -export function replicateBlobs(cm1, cm2) { - const { - rsm: [rsm1, rsm2], - destroy, - } = replicate(cm1, cm2) - rsm1.enableNamespace('blobIndex') - rsm1.enableNamespace('blob') - rsm2.enableNamespace('blobIndex') - rsm2.enableNamespace('blob') - return { - rsm: /** @type {const} */ ([rsm1, rsm2]), - destroy, - } -} - /** * @param {*} rs * @returns {Promise} From 420e21a933d4513a57f6dd64edc1ba54564f8225 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:34:17 +0900 Subject: [PATCH 58/69] fix discovery-key event --- src/local-peers.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index aef72c678..ad032d5ef 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -207,7 +207,7 @@ class Peer { * @property {(peers: PeerInfo[]) => void} peers Emitted whenever the connection status of peers changes. An array of peerInfo objects with a peer id and the peer connection status * @property {(peer: PeerInfoConnected) => void} peer-add Emitted when a new peer is connected * @property {(peerId: string, invite: InviteWithKeys) => void} invite Emitted when an invite is received - * @property {(discoveryKey: Buffer, stream: import('./types.js').ReplicationStream) => void} discovery-key Emitted when a new hypercore is replicated (by a peer) to a peer replication stream (passed as the second parameter) + * @property {(discoveryKey: Buffer, protomux: Protomux) => void} discovery-key Emitted when a new hypercore is replicated (by a peer) to a peer replication stream (passed as the second parameter) */ /** @extends {TypedEmitter} */ @@ -347,7 +347,7 @@ export class LocalPeers extends TypedEmitter { discoveryKey, stream.noiseStream.remotePublicKey ) - this.emit('discovery-key', discoveryKey, outerStream) + this.emit('discovery-key', discoveryKey, protomux) } ) From 5cd349d327bc39332422aef9630f4131cdbd97be Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:34:33 +0900 Subject: [PATCH 59/69] add coreCount to sync state --- src/sync/namespace-sync-state.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/sync/namespace-sync-state.js b/src/sync/namespace-sync-state.js index c69902ca9..81934e3a9 100644 --- a/src/sync/namespace-sync-state.js +++ b/src/sync/namespace-sync-state.js @@ -2,7 +2,7 @@ import { CoreSyncState } from './core-sync-state.js' import { discoveryKey } from 'hypercore-crypto' /** - * @typedef {Omit & { dataToSync: boolean }} SyncState + * @typedef {Omit & { dataToSync: boolean, coreCount: number }} SyncState */ /** @@ -11,6 +11,7 @@ import { discoveryKey } from 'hypercore-crypto' export class NamespaceSyncState { /** @type {Map} */ #coreStates = new Map() + #coreCount = 0 #handleUpdate #namespace /** @type {SyncState | null} */ @@ -56,6 +57,7 @@ export class NamespaceSyncState { /** @type {SyncState} */ const state = { dataToSync: false, + coreCount: this.#coreCount, localState: createState(), remoteStates: {}, } @@ -84,6 +86,7 @@ export class NamespaceSyncState { * @param {Buffer} coreKey */ #addCore(core, coreKey) { + this.#coreCount++ const discoveryId = discoveryKey(coreKey).toString('hex') this.#getCoreState(discoveryId).attachCore(core) } From 04520f651a64c46b54d2c14d9e9c4cf8339fcdef Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:35:43 +0900 Subject: [PATCH 60/69] test sync with blocked peer & fix bugs --- src/sync/peer-sync-controller.js | 8 +++- src/sync/sync-api.js | 29 ++++++++------ test-e2e/sync.js | 66 ++++++++++++++++++++++++++++++++ test-e2e/utils-new.js | 28 ++++++++++---- 4 files changed, 110 insertions(+), 21 deletions(-) diff --git a/src/sync/peer-sync-controller.js b/src/sync/peer-sync-controller.js index abc5227b8..d1d55aa09 100644 --- a/src/sync/peer-sync-controller.js +++ b/src/sync/peer-sync-controller.js @@ -34,7 +34,7 @@ export class PeerSyncController { /** * @param {object} opts - * @param {import("protomux")} opts.protomux + * @param {import("protomux")} opts.protomux * @param {import("../core-manager/index.js").CoreManager} opts.coreManager * @param {import("./sync-state.js").SyncState} opts.syncState * @param {import("../capabilities.js").Capabilities} opts.capabilities @@ -68,7 +68,11 @@ export class PeerSyncController { } get peerId() { - return this.peerKey?.toString('hex') + return this.peerKey.toString('hex') + } + + get syncCapability() { + return this.#syncCapability } /** diff --git a/src/sync/sync-api.js b/src/sync/sync-api.js index f2ebbe5bf..018de6e9b 100644 --- a/src/sync/sync-api.js +++ b/src/sync/sync-api.js @@ -6,6 +6,7 @@ import { } from './peer-sync-controller.js' import { Logger } from '../logger.js' import { NAMESPACES } from '../core-manager/index.js' +import { keyToId } from '../utils.js' export const kHandleDiscoveryKey = Symbol('handle discovery key') @@ -23,6 +24,8 @@ export class SyncApi extends TypedEmitter { #capabilities /** @type {Map} */ #peerSyncControllers = new Map() + /** @type {Set} */ + #peerIds = new Set() /** @type {Set<'local' | 'remote'>} */ #dataSyncEnabled = new Set() /** @type {Map>} */ @@ -51,8 +54,7 @@ export class SyncApi extends TypedEmitter { } /** @type {import('../local-peers.js').LocalPeersEvents['discovery-key']} */ - [kHandleDiscoveryKey](discoveryKey, stream) { - const protomux = stream.noiseStream.userData + [kHandleDiscoveryKey](discoveryKey, protomux) { const peerSyncController = this.#peerSyncControllers.get(protomux) if (peerSyncController) { peerSyncController.handleDiscoveryKey(discoveryKey) @@ -113,10 +115,10 @@ export class SyncApi extends TypedEmitter { async waitForSync(type) { const state = this.getState() const namespaces = type === 'initial' ? PRESYNC_NAMESPACES : NAMESPACES - if (isSynced(state, namespaces, this.#peerSyncControllers.size)) return + if (isSynced(state, namespaces, this.#peerSyncControllers)) return return new Promise((res) => { this.on('sync-state', function onState(state) { - if (!isSynced(state, namespaces, this.#peerSyncControllers.size)) return + if (!isSynced(state, namespaces, this.#peerSyncControllers)) return this.off('sync-state', onState) res(null) }) @@ -131,7 +133,7 @@ export class SyncApi extends TypedEmitter { * will then handle validation of role records to ensure that the peer is * actually still part of the project. * - * @param {{ protomux: import('protomux') }} peer + * @param {{ protomux: import('protomux') }} peer */ #handlePeerAdd = (peer) => { const { protomux } = peer @@ -150,6 +152,7 @@ export class SyncApi extends TypedEmitter { logger: this.#l, }) this.#peerSyncControllers.set(protomux, peerSyncController) + if (peerSyncController.peerId) this.#peerIds.add(peerSyncController.peerId) if (this.#dataSyncEnabled.has('local')) { peerSyncController.enableDataSync() @@ -170,7 +173,7 @@ export class SyncApi extends TypedEmitter { * Called when a peer is removed from the creator core, e.g. when the * connection is terminated. * - * @param {{ protomux: import('protomux') }} peer + * @param {{ protomux: import('protomux'), remotePublicKey: Buffer }} peer */ #handlePeerRemove = (peer) => { const { protomux } = peer @@ -182,6 +185,7 @@ export class SyncApi extends TypedEmitter { return } this.#peerSyncControllers.delete(protomux) + this.#peerIds.delete(keyToId(peer.remotePublicKey)) this.#pendingDiscoveryKeys.delete(protomux) } } @@ -191,15 +195,16 @@ export class SyncApi extends TypedEmitter { * * @param {import('./sync-state.js').State} state * @param {readonly import('../core-manager/index.js').Namespace[]} namespaces - * @param {number} peerCount + * @param {Map} peerSyncControllers */ -function isSynced(state, namespaces, peerCount) { +function isSynced(state, namespaces, peerSyncControllers) { for (const ns of namespaces) { if (state[ns].dataToSync) return false - const remoteStates = Object.values(state[ns].remoteStates) - if (remoteStates.length !== peerCount) return false - for (const rs of remoteStates) { - if (rs.status === 'connecting') return false + for (const psc of peerSyncControllers.values()) { + const { peerId } = psc + if (psc.syncCapability[ns] === 'blocked') continue + if (!(peerId in state[ns].remoteStates)) return false + if (state[ns].remoteStates[peerId].status === 'connecting') return false } } return true diff --git a/test-e2e/sync.js b/test-e2e/sync.js index d26f374d5..beaee3e62 100644 --- a/test-e2e/sync.js +++ b/test-e2e/sync.js @@ -14,6 +14,7 @@ import { PRESYNC_NAMESPACES } from '../src/sync/peer-sync-controller.js' import { generate } from '@mapeo/mock-data' import { valueOf } from '../src/utils.js' import pTimeout from 'p-timeout' +import { BLOCKED_ROLE_ID, COORDINATOR_ROLE_ID } from '../src/capabilities.js' const SCHEMAS_INITIAL_SYNC = ['preset', 'field'] @@ -188,3 +189,68 @@ test('shares cores', async function (t) { } } }) + +test('no sync capabilities === no namespaces sync apart from auth', async (t) => { + const COUNT = 3 + const managers = await createManagers(COUNT) + const [invitor, invitee, blocked] = managers + const disconnect1 = connectPeers(managers, { discovery: false }) + const projectId = await invitor.createProject() + await invite({ + invitor, + invitees: [blocked], + projectId, + roleId: BLOCKED_ROLE_ID, + }) + await invite({ + invitor, + invitees: [invitee], + projectId, + roleId: COORDINATOR_ROLE_ID, + }) + + const projects = await Promise.all( + managers.map((m) => m.getProject(projectId)) + ) + const [invitorProject, inviteeProject] = projects + + const generatedDocs = (await seedDatabases([inviteeProject])).flat() + const configDocsCount = generatedDocs.filter( + (doc) => doc.schemaName !== 'observation' + ).length + const dataDocsCount = generatedDocs.length - configDocsCount + + for (const project of projects) { + project.$sync.start() + } + + await waitForSync([inviteeProject, invitorProject], 'full') + + const [invitorState, inviteeState, blockedState] = projects.map((p) => + p.$sync.getState() + ) + + t.is(invitorState.config.localState.have, configDocsCount + COUNT) // count device info doc for each invited device + t.is(invitorState.data.localState.have, dataDocsCount) + t.is(blockedState.config.localState.have, 1) // just the device info doc + t.is(blockedState.data.localState.have, 0) // no data docs synced + + for (const ns of NAMESPACES) { + if (ns === 'auth') { + t.is(invitorState[ns].coreCount, 3) + t.is(inviteeState[ns].coreCount, 3) + t.is(blockedState[ns].coreCount, 3) + } else if (PRESYNC_NAMESPACES.includes(ns)) { + t.is(invitorState[ns].coreCount, 3) + t.is(inviteeState[ns].coreCount, 3) + t.is(blockedState[ns].coreCount, 1) + } else { + t.is(invitorState[ns].coreCount, 2) + t.is(inviteeState[ns].coreCount, 2) + t.is(blockedState[ns].coreCount, 1) + } + t.alike(invitorState[ns].localState, inviteeState[ns].localState) + } + + await disconnect1() +}) diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js index 7635549e1..ed2f82587 100644 --- a/test-e2e/utils-new.js +++ b/test-e2e/utils-new.js @@ -40,8 +40,7 @@ export function connectPeers(managers, { discovery = true } = {}) { const r1 = managers[i][kManagerReplicate](true) const r2 = managers[j][kManagerReplicate](false) replicationStreams.push(r1, r2) - // @ts-ignore - either the types or wrong, or we're returning the wrong thing - r1.rawStream.pipe(r2.rawStream).pipe(r1.rawStream) + r1.pipe(r2).pipe(r1) } } return function destroy() { @@ -197,23 +196,35 @@ export function round(value, decimalPlaces) { * number of peers to connect. * * @param {import('../src/mapeo-project.js').MapeoProject} project - * @param {number} peerCount + * @param {string[]} peerIds * @param {'initial' | 'full'} [type] */ -async function waitForProjectSync(project, peerCount, type = 'initial') { +async function waitForProjectSync(project, peerIds, type = 'initial') { const state = await project.$sync.getState() - if (Object.keys(state.auth.remoteStates).length === peerCount) { + if (hasPeerIds(state.auth.remoteStates, peerIds)) { return project.$sync.waitForSync(type) } return new Promise((res) => { project.$sync.on('sync-state', function onState(state) { - if (Object.keys(state.auth.remoteStates).length !== peerCount) return + if (!hasPeerIds(state.auth.remoteStates, peerIds)) return project.$sync.off('sync-state', onState) res(project.$sync.waitForSync(type)) }) }) } +/** + * @param {Record} remoteStates + * @param {string[]} peerIds + * @returns + */ +function hasPeerIds(remoteStates, peerIds) { + for (const peerId of peerIds) { + if (!(peerId in remoteStates)) return false + } + return true +} + /** * Wait for all projects to connect and sync * @@ -223,7 +234,10 @@ async function waitForProjectSync(project, peerCount, type = 'initial') { export function waitForSync(projects, type = 'initial') { return Promise.all( projects.map((project) => { - return waitForProjectSync(project, projects.length - 1, type) + const peerIds = projects + .filter((p) => p !== project) + .map((p) => p.deviceId) + return waitForProjectSync(project, peerIds, type) }) ) } From 53e388e9fdb19955e7046334ad3fc22bc9d0a150 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:36:02 +0900 Subject: [PATCH 61/69] fix datatype unit tests --- tests/data-type.js | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/tests/data-type.js b/tests/data-type.js index 14940bb28..2194f9cd9 100644 --- a/tests/data-type.js +++ b/tests/data-type.js @@ -84,7 +84,7 @@ test('test validity of `createdBy` field from another peer', async (t) => { const obs = await dt1.create(obsFixture) const driveId = ds1.writerCore.key - const { destroy } = replicateDataStore(cm1, cm2) + const { destroy } = replicate(cm1, cm2) await waitForCores(cm2, [driveId]) const replicatedCore = cm2.getCoreByKey(driveId) await replicatedCore.update({ wait: true }) @@ -135,17 +135,3 @@ async function testenv(opts) { return { coreManager, dataType, dataStore } } - -function replicateDataStore(cm1, cm2) { - const { - rsm: [rsm1, rsm2], - destroy, - } = replicate(cm1, cm2) - - rsm1.enableNamespace('data') - rsm2.enableNamespace('data') - return { - rsm: /** @type {const} */ ([rsm1, rsm2]), - destroy, - } -} From 41895a7fdb883415c9d35bac6a1057b99fb43148 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:36:44 +0900 Subject: [PATCH 62/69] fix blobs server unit tests --- tests/fastify-plugins/blobs.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/fastify-plugins/blobs.js b/tests/fastify-plugins/blobs.js index 5f72e33d4..84b257ddc 100644 --- a/tests/fastify-plugins/blobs.js +++ b/tests/fastify-plugins/blobs.js @@ -9,8 +9,11 @@ import fastify from 'fastify' import { BlobStore } from '../../src/blob-store/index.js' import BlobServerPlugin from '../../src/fastify-plugins/blobs.js' import { projectKeyToPublicId } from '../../src/utils.js' -import { replicateBlobs } from '../helpers/blob-store.js' -import { createCoreManager, waitForCores } from '../helpers/core-manager.js' +import { + createCoreManager, + waitForCores, + replicate, +} from '../helpers/core-manager.js' test('Plugin throws error if missing getBlobStore option', async (t) => { const server = fastify() @@ -224,7 +227,7 @@ test('GET photo returns 404 when trying to get non-replicated blob', async (t) = const [{ blobId }] = data - const { destroy } = replicateBlobs(cm1, cm2) + const { destroy } = replicate(cm1, cm2) await waitForCores(cm2, [cm1.getWriterCore('blobIndex').key]) From 7df2c73e1e7855faf3a49b7073e491adb6507dd6 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Mon, 20 Nov 2023 22:46:03 +0900 Subject: [PATCH 63/69] remote peer-sync-controller unit test This is now tested in e2e tests --- tests/sync/peer-sync-controller.js | 188 ----------------------------- 1 file changed, 188 deletions(-) delete mode 100644 tests/sync/peer-sync-controller.js diff --git a/tests/sync/peer-sync-controller.js b/tests/sync/peer-sync-controller.js deleted file mode 100644 index 5d9e2c220..000000000 --- a/tests/sync/peer-sync-controller.js +++ /dev/null @@ -1,188 +0,0 @@ -// @ts-check - -import test from 'brittle' -import Hypercore from 'hypercore' -import { PeerSyncController } from '../../src/sync/peer-sync-controller.js' -import { createCoreManager } from '../helpers/core-manager.js' -import { KeyManager } from '@mapeo/crypto' -import { once } from 'node:events' -import { setTimeout } from 'node:timers/promises' -import { NAMESPACES } from '../../src/core-manager/index.js' -import { SyncState } from '../../src/sync/sync-state.js' -import { - BLOCKED_ROLE_ID, - CREATOR_CAPABILITIES, - DEFAULT_CAPABILITIES, -} from '../../src/capabilities.js' - -test('auth, config and blobIndex enabled by default', async (t) => { - const { - coreManagers: [cm1, cm2], - } = await testenv(CREATOR_CAPABILITIES) - - const preSyncNamespaces = /** @type {const} */ ([ - 'auth', - 'config', - 'blobIndex', - ]) - - const peerAddPromises = [] - for (const ns of preSyncNamespaces) { - peerAddPromises.push( - once(cm1.getWriterCore(ns).core, 'peer-add'), - once(cm1.getWriterCore(ns).core, 'peer-add') - ) - } - await Promise.all(peerAddPromises) - t.pass('pre-sync cores connected') - - // Wait to give other namespaces a chance to connect (they shouldn't) - await setTimeout(500) - - for (const ns of NAMESPACES) { - for (const cm of [cm1, cm2]) { - const nsCores = cm.getCores(ns) - t.is( - nsCores.length, - includes(preSyncNamespaces, ns) ? 2 : 1, - 'preSync namespaces have 2 cores, others have 1' - ) - for (const { core } of nsCores) { - if (includes(preSyncNamespaces, ns)) { - t.is(core.peers.length, 1, 'pre-sync namespace cores have one peer') - } else { - t.is(core.peers.length, 0, 'non-pre-sync cores have no peers') - } - } - } - } -}) - -test('enabling data sync replicates all cores', async (t) => { - const { - coreManagers: [cm1, cm2], - peerSyncControllers: [psc1, psc2], - } = await testenv(CREATOR_CAPABILITIES) - - psc1.enableDataSync() - psc2.enableDataSync() - - const peerAddPromises = [] - for (const ns of NAMESPACES) { - peerAddPromises.push( - once(cm1.getWriterCore(ns).core, 'peer-add'), - once(cm1.getWriterCore(ns).core, 'peer-add') - ) - } - await Promise.all(peerAddPromises) - - for (const ns of NAMESPACES) { - for (const [i, cm] of [cm1, cm2].entries()) { - const nsCores = cm.getCores(ns) - t.is(nsCores.length, 2, `cm${i + 1}: namespace ${ns} has 2 cores now`) - for (const { core } of nsCores) { - t.is( - core.peers.length, - 1, - `cm${i + 1}: ${ns} ${ - core === cm.getWriterCore(ns).core ? 'own' : 'synced' - } core is connected` - ) - } - } - } -}) - -test('no sync capabilities === no namespaces sync apart from auth', async (t) => { - const { - coreManagers: [cm1, cm2], - peerSyncControllers: [psc1, psc2], - } = await testenv(DEFAULT_CAPABILITIES[BLOCKED_ROLE_ID]) - - psc1.enableDataSync() - psc2.enableDataSync() - - // Wait to give cores a chance to connect - await setTimeout(500) - - for (const ns of NAMESPACES) { - for (const cm of [cm1, cm2]) { - const nsCores = cm.getCores(ns) - if (ns === 'auth') { - t.is(nsCores.length, 2, `all auth cores have been shared`) - // no guarantees about sharing of other cores yet - } - for (const { core } of nsCores) { - const isCreatorCore = core === cm.creatorCore - if (isCreatorCore) { - t.is(core.peers.length, 1, 'creator core remains connected') - } else { - t.is(core.peers.length, 0, 'core is disconnected') - } - } - } - } -}) - -/** - * - * @param {import('../../src/capabilities.js').Capability} cap - * @returns - */ -async function testenv(cap) { - const { publicKey: projectKey, secretKey: projectSecretKey } = - KeyManager.generateProjectKeypair() - const cm1 = await createCoreManager({ projectKey, projectSecretKey }) - const cm2 = await createCoreManager({ projectKey }) - - const stream1 = Hypercore.createProtocolStream(true, { - ondiscoverykey: (discoveryKey) => - cm1.handleDiscoveryKey(discoveryKey, stream1), - }) - const stream2 = Hypercore.createProtocolStream(false, { - ondiscoverykey: (discoveryKey) => - cm2.handleDiscoveryKey(discoveryKey, stream2), - }) - stream1.pipe(stream2).pipe(stream1) - - const psc1 = new PeerSyncController({ - protomux: stream1.noiseStream.userData, - coreManager: cm1, - syncState: new SyncState({ coreManager: cm1 }), - // @ts-expect-error - capabilities: { - async getCapabilities() { - return cap - }, - }, - }) - const psc2 = new PeerSyncController({ - protomux: stream2.noiseStream.userData, - coreManager: cm2, - syncState: new SyncState({ coreManager: cm2 }), - // @ts-expect-error - capabilities: { - async getCapabilities() { - return cap - }, - }, - }) - - return { - peerSyncControllers: [psc1, psc2], - coreManagers: [cm1, cm2], - } -} - -/** - * Helper for Typescript array.prototype.includes - * - * @template {U} T - * @template U - * @param {ReadonlyArray} coll - * @param {U} el - * @returns {el is T} - */ -function includes(coll, el) { - return coll.includes(/** @type {T} */ (el)) -} From 1133e75f6a9ccc4bb2ee8b97a00aec96957f2d53 Mon Sep 17 00:00:00 2001 From: Andrew Chou Date: Mon, 20 Nov 2023 10:48:06 -0500 Subject: [PATCH 64/69] fix type issues caused by bad lockfile --- package-lock.json | 1000 +++------------------------------------------ 1 file changed, 62 insertions(+), 938 deletions(-) diff --git a/package-lock.json b/package-lock.json index 863c51fd7..51ef6996c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,6 @@ "@mapeo/sqlite-indexer": "1.0.0-alpha.8", "@sinclair/typebox": "^0.29.6", "b4a": "^1.6.3", - "base32.js": "^0.1.0", "better-sqlite3": "^8.7.0", "big-sparse-array": "^1.0.3", "bogon": "^1.1.0", @@ -27,13 +26,13 @@ "corestore": "^6.8.4", "debug": "^4.3.4", "drizzle-orm": "0.28.2", - "eventemitter3": "^5.0.1", "fastify": ">= 4", "fastify-plugin": "^4.5.0", + "hyperblobs": "2.3.0", "hypercore": "10.17.0", - "hypercore-crypto": "^3.3.1", - "hyperdrive": "^11.5.3", - "hyperswarm": "^4.4.1", + "hypercore-crypto": "3.4.0", + "hyperdrive": "11.5.3", + "hyperswarm": "4.4.1", "magic-bytes.js": "^1.0.14", "map-obj": "^5.0.2", "multi-core-indexer": "1.0.0-alpha.7", @@ -49,8 +48,7 @@ "throttle-debounce": "^5.0.0", "tiny-typed-emitter": "^2.1.0", "type-fest": "^4.5.0", - "varint": "^6.0.0", - "z32": "^1.0.1" + "varint": "^6.0.0" }, "devDependencies": { "@bufbuild/buf": "^1.26.1", @@ -69,10 +67,8 @@ "brittle": "^3.2.1", "cpy": "^10.1.0", "cpy-cli": "^5.0.0", - "depcheck": "^1.4.3", "drizzle-kit": "^0.19.12", "eslint": "^8.39.0", - "fastify": "^4.24.3", "husky": "^8.0.0", "light-my-request": "^5.10.0", "lint-staged": "^14.0.1", @@ -94,176 +90,31 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.22.13", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.22.13.tgz", - "integrity": "sha512-XktuhWlJ5g+3TJXc5upd9Ks1HutSArik6jf2eAjYFyIOf4ej3RN+184cZbzDvbPnuTJIUhPKKJE3cIsYTiAT3w==", - "dev": true, - "dependencies": { - "@babel/highlight": "^7.22.13", - "chalk": "^2.4.2" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/code-frame/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/code-frame/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true - }, - "node_modules/@babel/code-frame/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/code-frame/node_modules/has-flag": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/code-frame/node_modules/supports-color": { - "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", - "dev": true, - "dependencies": { - "has-flag": "^3.0.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/generator": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.23.3.tgz", - "integrity": "sha512-keeZWAV4LU3tW0qRi19HRpabC/ilM0HRBBzf9/k8FFiG4KVpiv0FIy4hHfLfFQZNhziCTPTmd59zoyv6DNISzg==", - "dev": true, - "dependencies": { - "@babel/types": "^7.23.3", - "@jridgewell/gen-mapping": "^0.3.2", - "@jridgewell/trace-mapping": "^0.3.17", - "jsesc": "^2.5.1" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-environment-visitor": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-environment-visitor/-/helper-environment-visitor-7.22.20.tgz", - "integrity": "sha512-zfedSIzFhat/gFhWfHtgWvlec0nqB9YEIVrpuwjruLlXfUSnA8cJB0miHKwqDnQ7d32aKo2xt88/xZptwxbfhA==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-function-name": { - "version": "7.23.0", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.23.0.tgz", - "integrity": "sha512-OErEqsrxjZTJciZ4Oo+eoZqeW9UIiOcuYKRJA4ZAgV9myA+pOXhhmpfNCKjEH/auVfEYVFJ6y1Tc4r0eIApqiw==", - "dev": true, - "dependencies": { - "@babel/template": "^7.22.15", - "@babel/types": "^7.23.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-hoist-variables": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.22.5.tgz", - "integrity": "sha512-wGjk9QZVzvknA6yKIUURb8zY3grXCcOZt+/7Wcy8O2uctxhplmUPkOdlgoNhmdVee2c92JXbf1xpMtVNbfoxRw==", - "dev": true, - "dependencies": { - "@babel/types": "^7.22.5" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-split-export-declaration": { - "version": "7.22.6", - "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.22.6.tgz", - "integrity": "sha512-AsUnxuLhRYsisFiaJwvp1QF+I3KjD5FOxut14q/GzovUe6orHLesW2C7d754kRm53h5gqrz6sFl6sxc4BVtE/g==", + "version": "7.18.6", "dev": true, + "license": "MIT", "dependencies": { - "@babel/types": "^7.22.5" + "@babel/highlight": "^7.18.6" }, "engines": { "node": ">=6.9.0" } }, - "node_modules/@babel/helper-string-parser": { - "version": "7.22.5", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.22.5.tgz", - "integrity": "sha512-mM4COjgZox8U+JcXQwPijIZLElkgEpO5rsERVDJTc2qfCDfERyob6k5WegS14SX18IIjv+XD+GrqNumY5JRCDw==", - "dev": true, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.22.20.tgz", - "integrity": "sha512-Y4OZ+ytlatR8AI+8KZfKuL5urKp7qey08ha31L8b3BwewJAoJamTzyvxPR/5D+KkdJCGPq/+8TukHBlY10FX9A==", + "version": "7.19.1", "dev": true, + "license": "MIT", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/highlight": { - "version": "7.22.20", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.22.20.tgz", - "integrity": "sha512-dkdMCN3py0+ksCgYmGG8jKeGA/8Tk+gJwSYYlFGxG5lmhfKNoAy004YpLxpS1W2J8m/EK2Ew+yOs9pVRwO89mg==", + "version": "7.18.6", "dev": true, + "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.22.20", - "chalk": "^2.4.2", + "@babel/helper-validator-identifier": "^7.18.6", + "chalk": "^2.0.0", "js-tokens": "^4.0.0" }, "engines": { @@ -272,9 +123,8 @@ }, "node_modules/@babel/highlight/node_modules/ansi-styles": { "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", "dev": true, + "license": "MIT", "dependencies": { "color-convert": "^1.9.0" }, @@ -284,9 +134,8 @@ }, "node_modules/@babel/highlight/node_modules/chalk": { "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", "dev": true, + "license": "MIT", "dependencies": { "ansi-styles": "^3.2.1", "escape-string-regexp": "^1.0.5", @@ -298,42 +147,37 @@ }, "node_modules/@babel/highlight/node_modules/color-convert": { "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", "dev": true, + "license": "MIT", "dependencies": { "color-name": "1.1.3" } }, "node_modules/@babel/highlight/node_modules/color-name": { "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/@babel/highlight/node_modules/escape-string-regexp": { "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", "dev": true, + "license": "MIT", "engines": { "node": ">=0.8.0" } }, "node_modules/@babel/highlight/node_modules/has-flag": { "version": "3.0.0", - "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", - "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", "dev": true, + "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/@babel/highlight/node_modules/supports-color": { "version": "5.5.0", - "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", - "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", "dev": true, + "license": "MIT", "dependencies": { "has-flag": "^3.0.0" }, @@ -341,76 +185,6 @@ "node": ">=4" } }, - "node_modules/@babel/parser": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.23.3.tgz", - "integrity": "sha512-uVsWNvlVsIninV2prNz/3lHCb+5CJ+e+IUBfbjToAHODtfGYLfCFuY4AU7TskI+dAKk+njsPiBjq1gKTvZOBaw==", - "dev": true, - "bin": { - "parser": "bin/babel-parser.js" - }, - "engines": { - "node": ">=6.0.0" - } - }, - "node_modules/@babel/template": { - "version": "7.22.15", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.22.15.tgz", - "integrity": "sha512-QPErUVm4uyJa60rkI73qneDacvdvzxshT3kksGqlGWYdOTIUOwJ7RDUL8sGqslY1uXWSL6xMFKEXDS3ox2uF0w==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/parser": "^7.22.15", - "@babel/types": "^7.22.15" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.23.3.tgz", - "integrity": "sha512-+K0yF1/9yR0oHdE0StHuEj3uTPzwwbrLGfNOndVJVV2TqA5+j3oljJUb4nmB954FLGjNem976+B+eDuLIjesiQ==", - "dev": true, - "dependencies": { - "@babel/code-frame": "^7.22.13", - "@babel/generator": "^7.23.3", - "@babel/helper-environment-visitor": "^7.22.20", - "@babel/helper-function-name": "^7.23.0", - "@babel/helper-hoist-variables": "^7.22.5", - "@babel/helper-split-export-declaration": "^7.22.6", - "@babel/parser": "^7.23.3", - "@babel/types": "^7.23.3", - "debug": "^4.1.0", - "globals": "^11.1.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/traverse/node_modules/globals": { - "version": "11.12.0", - "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", - "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/types": { - "version": "7.23.3", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.23.3.tgz", - "integrity": "sha512-OZnvoH2l8PK5eUvEcUyCt/sXgr/h+UWpVuBbOljwcrAgUl6lpchoQ++PHGyQy1AtYnVA6CEq3y5xeEI10brpXw==", - "dev": true, - "dependencies": { - "@babel/helper-string-parser": "^7.22.5", - "@babel/helper-validator-identifier": "^7.22.20", - "to-fast-properties": "^2.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@bcoe/v8-coverage": { "version": "0.2.3", "dev": true, @@ -623,7 +397,6 @@ }, "node_modules/@fastify/ajv-compiler": { "version": "3.5.0", - "dev": true, "license": "MIT", "dependencies": { "ajv": "^8.11.0", @@ -633,7 +406,6 @@ }, "node_modules/@fastify/ajv-compiler/node_modules/ajv": { "version": "8.12.0", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -648,7 +420,6 @@ }, "node_modules/@fastify/ajv-compiler/node_modules/json-schema-traverse": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/@fastify/busboy": { @@ -662,18 +433,15 @@ }, "node_modules/@fastify/deepmerge": { "version": "1.3.0", - "dev": true, "license": "MIT" }, "node_modules/@fastify/error": { "version": "3.4.0", "resolved": "https://registry.npmjs.org/@fastify/error/-/error-3.4.0.tgz", - "integrity": "sha512-e/mafFwbK3MNqxUcFBLgHhgxsF8UT1m8aj0dAlqEa2nJEgPsRtpHTZ3ObgrgkZ2M1eJHPTwgyUl/tXkvabsZdQ==", - "dev": true + "integrity": "sha512-e/mafFwbK3MNqxUcFBLgHhgxsF8UT1m8aj0dAlqEa2nJEgPsRtpHTZ3ObgrgkZ2M1eJHPTwgyUl/tXkvabsZdQ==" }, "node_modules/@fastify/fast-json-stringify-compiler": { "version": "4.3.0", - "dev": true, "license": "MIT", "dependencies": { "fast-json-stringify": "^5.7.0" @@ -831,20 +599,6 @@ "node": ">=8" } }, - "node_modules/@jridgewell/gen-mapping": { - "version": "0.3.3", - "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz", - "integrity": "sha512-HLhSWOLRi875zjjMG/r+Nv0oCW8umGb0BgEhyX3dDX3egwZtB8PqLnjz3yedt8R5StBrzcg4aBpnh8UA9D1BoQ==", - "dev": true, - "dependencies": { - "@jridgewell/set-array": "^1.0.1", - "@jridgewell/sourcemap-codec": "^1.4.10", - "@jridgewell/trace-mapping": "^0.3.9" - }, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.0", "dev": true, @@ -853,15 +607,6 @@ "node": ">=6.0.0" } }, - "node_modules/@jridgewell/set-array": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@jridgewell/set-array/-/set-array-1.1.2.tgz", - "integrity": "sha512-xnkseuNADM0gt2bs+BvhO0p78Mk762YnZdsuzFV018NoG1Sj1SCQvpSqa7XUaTam5vAGasABV9qXASMKnFMwMw==", - "dev": true, - "engines": { - "node": ">=6.0.0" - } - }, "node_modules/@jridgewell/sourcemap-codec": { "version": "1.4.14", "dev": true, @@ -1183,12 +928,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/minimatch": { - "version": "3.0.5", - "resolved": "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz", - "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", - "dev": true - }, "node_modules/@types/minimist": { "version": "1.2.2", "dev": true, @@ -1208,12 +947,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/parse-json": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", - "integrity": "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==", - "dev": true - }, "node_modules/@types/sinonjs__fake-timers": { "version": "8.1.2", "dev": true, @@ -1241,75 +974,6 @@ "@types/node": "*" } }, - "node_modules/@vue/compiler-core": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/compiler-core/-/compiler-core-3.3.8.tgz", - "integrity": "sha512-hN/NNBUECw8SusQvDSqqcVv6gWq8L6iAktUR0UF3vGu2OhzRqcOiAno0FmBJWwxhYEXRlQJT5XnoKsVq1WZx4g==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.23.0", - "@vue/shared": "3.3.8", - "estree-walker": "^2.0.2", - "source-map-js": "^1.0.2" - } - }, - "node_modules/@vue/compiler-dom": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/compiler-dom/-/compiler-dom-3.3.8.tgz", - "integrity": "sha512-+PPtv+p/nWDd0AvJu3w8HS0RIm/C6VGBIRe24b9hSyNWOAPEUosFZ5diwawwP8ip5sJ8n0Pe87TNNNHnvjs0FQ==", - "dev": true, - "dependencies": { - "@vue/compiler-core": "3.3.8", - "@vue/shared": "3.3.8" - } - }, - "node_modules/@vue/compiler-sfc": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/compiler-sfc/-/compiler-sfc-3.3.8.tgz", - "integrity": "sha512-WMzbUrlTjfYF8joyT84HfwwXo+8WPALuPxhy+BZ6R4Aafls+jDBnSz8PDz60uFhuqFbl3HxRfxvDzrUf3THwpA==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.23.0", - "@vue/compiler-core": "3.3.8", - "@vue/compiler-dom": "3.3.8", - "@vue/compiler-ssr": "3.3.8", - "@vue/reactivity-transform": "3.3.8", - "@vue/shared": "3.3.8", - "estree-walker": "^2.0.2", - "magic-string": "^0.30.5", - "postcss": "^8.4.31", - "source-map-js": "^1.0.2" - } - }, - "node_modules/@vue/compiler-ssr": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/compiler-ssr/-/compiler-ssr-3.3.8.tgz", - "integrity": "sha512-hXCqQL/15kMVDBuoBYpUnSYT8doDNwsjvm3jTefnXr+ytn294ySnT8NlsFHmTgKNjwpuFy7XVV8yTeLtNl/P6w==", - "dev": true, - "dependencies": { - "@vue/compiler-dom": "3.3.8", - "@vue/shared": "3.3.8" - } - }, - "node_modules/@vue/reactivity-transform": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/reactivity-transform/-/reactivity-transform-3.3.8.tgz", - "integrity": "sha512-49CvBzmZNtcHua0XJ7GdGifM8GOXoUMOX4dD40Y5DxI3R8OUhMlvf2nvgUAcPxaXiV5MQQ1Nwy09ADpnLQUqRw==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.23.0", - "@vue/compiler-core": "3.3.8", - "@vue/shared": "3.3.8", - "estree-walker": "^2.0.2", - "magic-string": "^0.30.5" - } - }, - "node_modules/@vue/shared": { - "version": "3.3.8", - "resolved": "https://registry.npmjs.org/@vue/shared/-/shared-3.3.8.tgz", - "integrity": "sha512-8PGwybFwM4x8pcfgqEQFy70NaQxASvOC5DJwLQfpArw1UDfUXrJkdxD3BhVTMS+0Lef/TU7YO0Jvr0jJY8T+mw==", - "dev": true - }, "node_modules/@yarnpkg/lockfile": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@yarnpkg/lockfile/-/lockfile-1.1.0.tgz", @@ -1319,7 +983,6 @@ "version": "3.0.0", "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", - "dev": true, "dependencies": { "event-target-shim": "^5.0.0" }, @@ -1329,7 +992,6 @@ }, "node_modules/abstract-logging": { "version": "2.0.1", - "dev": true, "license": "MIT" }, "node_modules/acorn": { @@ -1467,7 +1129,6 @@ }, "node_modules/archy": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/argparse": { @@ -1487,29 +1148,11 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/array-differ": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/array-differ/-/array-differ-3.0.0.tgz", - "integrity": "sha512-THtfYS6KtME/yIAhKjZ2ul7XI96lQGHRputJQHO80LAWQnuGP4iCIN8vdMRboGbIEYBwU33q8Tch1os2+X0kMg==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/array-flatten": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-2.1.2.tgz", "integrity": "sha512-hNfzcOV8W4NdualtqBFPyVO+54DSJuZGY9qT4pRroB6S9e3iiido2ISIC5h9R2sPJ8H3FHCIiEnsv1lPXO3KtQ==" }, - "node_modules/array-union": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz", - "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/arraybuffer.prototype.slice": { "version": "1.0.1", "dev": true, @@ -1529,15 +1172,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/arrify": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", - "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", - "dev": true, - "engines": { - "node": ">=8" - } - }, "node_modules/at-least-node": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/at-least-node/-/at-least-node-1.0.0.tgz", @@ -1550,7 +1184,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/atomic-sleep/-/atomic-sleep-1.0.0.tgz", "integrity": "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==", - "dev": true, "engines": { "node": ">=8.0.0" } @@ -1568,7 +1201,6 @@ }, "node_modules/avvio": { "version": "8.2.1", - "dev": true, "license": "MIT", "dependencies": { "archy": "^1.0.0", @@ -1839,15 +1471,6 @@ "integrity": "sha512-HpX65o1Hnr9HH25ojC1YGs7HCQLq0GCOibSaWER0eNpgJ/Z1MZv2mTc7+xh6WOPxbRVcmgbv4hGU+uSQ/2xFZQ==", "dev": true }, - "node_modules/callsite": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/callsite/-/callsite-1.0.0.tgz", - "integrity": "sha512-0vdNRFXn5q+dtOqjfFtmtlI9N2eVZ7LMyEV2iKC5mEEFvSg/69Ml6b/WU2qF8W1nLRa0wiSrDT3Y5jOHZCwKPQ==", - "dev": true, - "engines": { - "node": "*" - } - }, "node_modules/callsites": { "version": "3.1.0", "dev": true, @@ -1856,18 +1479,6 @@ "node": ">=6" } }, - "node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/camelcase-keys": { "version": "8.0.2", "dev": true, @@ -2163,7 +1774,6 @@ }, "node_modules/cookie": { "version": "0.5.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -2183,22 +1793,6 @@ "xache": "^1.1.0" } }, - "node_modules/cosmiconfig": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-7.1.0.tgz", - "integrity": "sha512-AdmX6xUzdNASswsFtmwSt7Vj8po9IuqXm0UXz7QKPuEUmPB4XyjGfaAr2PSuELMwkRMVH1EpIkX5bTZGRB3eCA==", - "dev": true, - "dependencies": { - "@types/parse-json": "^4.0.0", - "import-fresh": "^3.2.1", - "parse-json": "^5.0.0", - "path-type": "^4.0.0", - "yaml": "^1.10.0" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/cp-file": { "version": "10.0.0", "dev": true, @@ -2417,152 +2011,30 @@ } }, "node_modules/deep-extend": { - "version": "0.6.0", - "license": "MIT", - "engines": { - "node": ">=4.0.0" - } - }, - "node_modules/deep-is": { - "version": "0.1.4", - "dev": true, - "license": "MIT" - }, - "node_modules/define-properties": { - "version": "1.2.0", - "dev": true, - "license": "MIT", - "dependencies": { - "has-property-descriptors": "^1.0.0", - "object-keys": "^1.1.1" - }, - "engines": { - "node": ">= 0.4" - }, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, - "node_modules/depcheck": { - "version": "1.4.7", - "resolved": "https://registry.npmjs.org/depcheck/-/depcheck-1.4.7.tgz", - "integrity": "sha512-1lklS/bV5chOxwNKA/2XUUk/hPORp8zihZsXflr8x0kLwmcZ9Y9BsS6Hs3ssvA+2wUVbG0U2Ciqvm1SokNjPkA==", - "dev": true, - "dependencies": { - "@babel/parser": "^7.23.0", - "@babel/traverse": "^7.23.2", - "@vue/compiler-sfc": "^3.3.4", - "callsite": "^1.0.0", - "camelcase": "^6.3.0", - "cosmiconfig": "^7.1.0", - "debug": "^4.3.4", - "deps-regex": "^0.2.0", - "findup-sync": "^5.0.0", - "ignore": "^5.2.4", - "is-core-module": "^2.12.0", - "js-yaml": "^3.14.1", - "json5": "^2.2.3", - "lodash": "^4.17.21", - "minimatch": "^7.4.6", - "multimatch": "^5.0.0", - "please-upgrade-node": "^3.2.0", - "readdirp": "^3.6.0", - "require-package-name": "^2.0.1", - "resolve": "^1.22.3", - "resolve-from": "^5.0.0", - "semver": "^7.5.4", - "yargs": "^16.2.0" - }, - "bin": { - "depcheck": "bin/depcheck.js" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/depcheck/node_modules/argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "dev": true, - "dependencies": { - "sprintf-js": "~1.0.2" - } - }, - "node_modules/depcheck/node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", - "dev": true, - "dependencies": { - "balanced-match": "^1.0.0" - } - }, - "node_modules/depcheck/node_modules/js-yaml": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.1.tgz", - "integrity": "sha512-okMH7OXXJ7YrN9Ok3/SXrnu4iX9yOk+25nqX4imS2npuvTYDmo/QEZoqwZkYaIDk3jVvBOTOIEgEhaLOynBS9g==", - "dev": true, - "dependencies": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - }, - "bin": { - "js-yaml": "bin/js-yaml.js" - } - }, - "node_modules/depcheck/node_modules/minimatch": { - "version": "7.4.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.6.tgz", - "integrity": "sha512-sBz8G/YjVniEz6lKPNpKxXwazJe4c19fEfV2GDMX6AjFz+MX9uDWIZW8XreVhkFW3fkIdTv/gxWr/Kks5FFAVw==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/depcheck/node_modules/resolve-from": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/resolve-from/-/resolve-from-5.0.0.tgz", - "integrity": "sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw==", - "dev": true, - "engines": { - "node": ">=8" - } - }, - "node_modules/depcheck/node_modules/semver": { - "version": "7.5.4", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", - "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", - "dev": true, - "dependencies": { - "lru-cache": "^6.0.0" - }, - "bin": { - "semver": "bin/semver.js" - }, + "version": "0.6.0", + "license": "MIT", "engines": { - "node": ">=10" + "node": ">=4.0.0" } }, - "node_modules/deps-regex": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/deps-regex/-/deps-regex-0.2.0.tgz", - "integrity": "sha512-PwuBojGMQAYbWkMXOY9Pd/NWCDNHVH12pnS7WHqZkTSeMESe4hwnKKRp0yR87g37113x4JPbo/oIvXY+s/f56Q==", - "dev": true + "node_modules/deep-is": { + "version": "0.1.4", + "dev": true, + "license": "MIT" }, - "node_modules/detect-file": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/detect-file/-/detect-file-1.0.0.tgz", - "integrity": "sha512-DtCOLG98P007x7wiiOmfI0fi3eIKyWiLTGJ2MDnVi/E04lWGbf+JzrRHMm0rgIIZJGtHpKpbVgLWHrv8xXpc3Q==", + "node_modules/define-properties": { + "version": "1.2.0", "dev": true, + "license": "MIT", + "dependencies": { + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, "engines": { - "node": ">=0.10.0" + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" } }, "node_modules/detect-libc": { @@ -3176,12 +2648,6 @@ "node": ">=4.0" } }, - "node_modules/estree-walker": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-2.0.2.tgz", - "integrity": "sha512-Rfkk/Mp/DL7JVje3u18FxFujQlTNR2q6QfMSMB7AvCBx91NGj/ba3kCfza0f6dVDbw7YlRf/nDrn7pQrCCyQ/w==", - "dev": true - }, "node_modules/esutils": { "version": "2.0.3", "dev": true, @@ -3203,7 +2669,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "dev": true, "engines": { "node": ">=6" } @@ -3211,7 +2676,8 @@ "node_modules/eventemitter3": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", - "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", + "dev": true }, "node_modules/events": { "version": "3.3.0", @@ -3250,18 +2716,6 @@ "node": ">=6" } }, - "node_modules/expand-tilde": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz", - "integrity": "sha512-A5EmesHW6rfnZ9ysHQjPdJRni0SRar0tjtG5MNtm9n5TUvsYU8oozprtRD4AqHxcZWWlVuAmQo2nWKfN9oyjTw==", - "dev": true, - "dependencies": { - "homedir-polyfill": "^1.0.1" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/ext": { "version": "1.7.0", "dev": true, @@ -3278,14 +2732,12 @@ "node_modules/fast-content-type-parse": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/fast-content-type-parse/-/fast-content-type-parse-1.1.0.tgz", - "integrity": "sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==", - "dev": true + "integrity": "sha512-fBHHqSTFLVnR61C+gltJuE5GkVQMV0S2nqUO8TJ+5Z3qAKG8vAx4FKai1s5jq/inV1+sREynIWSuQ6HgoSXpDQ==" }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", - "integrity": "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==", - "dev": true + "integrity": "sha512-WKgKWg5eUxvRZGwW8FvfbaH7AXSh2cL+3j5fMGzUMCxWBJ3dV3a7Wz8y2f/uQ0e3B6WmodD3oS54jTQ9HVTIIg==" }, "node_modules/fast-deep-equal": { "version": "3.1.3", @@ -3330,7 +2782,6 @@ "version": "5.8.0", "resolved": "https://registry.npmjs.org/fast-json-stringify/-/fast-json-stringify-5.8.0.tgz", "integrity": "sha512-VVwK8CFMSALIvt14U8AvrSzQAwN/0vaVRiFFUVlpnXSnDGrSkOAO5MtzyN8oQNjLd5AqTW5OZRgyjoNuAuR3jQ==", - "dev": true, "dependencies": { "@fastify/deepmerge": "^1.0.0", "ajv": "^8.10.0", @@ -3342,7 +2793,6 @@ }, "node_modules/fast-json-stringify/node_modules/ajv": { "version": "8.12.0", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.1", @@ -3357,7 +2807,6 @@ }, "node_modules/fast-json-stringify/node_modules/json-schema-traverse": { "version": "1.0.0", - "dev": true, "license": "MIT" }, "node_modules/fast-levenshtein": { @@ -3369,7 +2818,6 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/fast-querystring/-/fast-querystring-1.1.2.tgz", "integrity": "sha512-g6KuKWmFXc0fID8WWH0jit4g0AGBoJhCkJMb1RmbsSEUNvQ+ZC8D6CUZ+GtF8nMzSPXnhiePyyqqipzNNEnHjg==", - "dev": true, "dependencies": { "fast-decode-uri-component": "^1.0.1" } @@ -3378,7 +2826,6 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/fast-redact/-/fast-redact-3.3.0.tgz", "integrity": "sha512-6T5V1QK1u4oF+ATxs1lWUmlEk6P2T9HqJG3e2DnHOdVgZy2rFJBoEnrIedcTXlkAHU/zKC+7KETJ+KGGKwxgMQ==", - "dev": true, "engines": { "node": ">=6" } @@ -3389,14 +2836,12 @@ }, "node_modules/fast-uri": { "version": "2.2.0", - "dev": true, "license": "MIT" }, "node_modules/fastify": { "version": "4.24.3", "resolved": "https://registry.npmjs.org/fastify/-/fastify-4.24.3.tgz", "integrity": "sha512-6HHJ+R2x2LS3y1PqxnwEIjOTZxFl+8h4kSC/TuDPXtA+v2JnV9yEtOsNSKK1RMD7sIR2y1ZsA4BEFaid/cK5pg==", - "dev": true, "dependencies": { "@fastify/ajv-compiler": "^3.5.0", "@fastify/error": "^3.4.0", @@ -3422,7 +2867,6 @@ }, "node_modules/fastify/node_modules/semver": { "version": "7.5.4", - "dev": true, "license": "ISC", "dependencies": { "lru-cache": "^6.0.0" @@ -3436,7 +2880,6 @@ }, "node_modules/fastq": { "version": "1.14.0", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -3471,7 +2914,6 @@ "version": "7.7.0", "resolved": "https://registry.npmjs.org/find-my-way/-/find-my-way-7.7.0.tgz", "integrity": "sha512-+SrHpvQ52Q6W9f3wJoJBbAQULJuNEEQwBvlvYwACDhBTLOTMiQ0HYWh4+vC3OivGP2ENcTI1oKlFA2OepJNjhQ==", - "dev": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-querystring": "^1.0.0", @@ -3504,21 +2946,6 @@ "micromatch": "^4.0.2" } }, - "node_modules/findup-sync": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-5.0.0.tgz", - "integrity": "sha512-MzwXju70AuyflbgeOhzvQWAvvQdo1XL0A9bVvlXsYcFEBM87WR4OakL4OfZq+QRmr+duJubio+UtNQCPsVESzQ==", - "dev": true, - "dependencies": { - "detect-file": "^1.0.0", - "is-glob": "^4.0.3", - "micromatch": "^4.0.4", - "resolve-dir": "^1.0.1" - }, - "engines": { - "node": ">= 10.13.0" - } - }, "node_modules/flat-cache": { "version": "3.0.4", "dev": true, @@ -3601,7 +3028,6 @@ }, "node_modules/forwarded": { "version": "0.2.0", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.6" @@ -3640,13 +3066,9 @@ "license": "ISC" }, "node_modules/function-bind": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", - "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "version": "1.1.1", "dev": true, - "funding": { - "url": "https://github.com/sponsors/ljharb" - } + "license": "MIT" }, "node_modules/function.prototype.name": { "version": "1.1.5", @@ -3785,48 +3207,6 @@ "node": ">=10" } }, - "node_modules/global-modules": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/global-modules/-/global-modules-1.0.0.tgz", - "integrity": "sha512-sKzpEkf11GpOFuw0Zzjzmt4B4UZwjOcG757PPvrfhxcLFbq0wpsgpOqxpxtxFiCG4DtG93M6XRVbF2oGdev7bg==", - "dev": true, - "dependencies": { - "global-prefix": "^1.0.1", - "is-windows": "^1.0.1", - "resolve-dir": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/global-prefix": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/global-prefix/-/global-prefix-1.0.2.tgz", - "integrity": "sha512-5lsx1NUDHtSjfg0eHlmYvZKv8/nVqX4ckFbM+FrGcQ+04KWcWFo9P5MxPZYSzUvyzmdTbI7Eix8Q4IbELDqzKg==", - "dev": true, - "dependencies": { - "expand-tilde": "^2.0.2", - "homedir-polyfill": "^1.0.1", - "ini": "^1.3.4", - "is-windows": "^1.0.1", - "which": "^1.2.14" - }, - "engines": { - "node": ">=0.10.0" - } - }, - "node_modules/global-prefix/node_modules/which": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/which/-/which-1.3.1.tgz", - "integrity": "sha512-HxJdYWq1MTIQbJ3nw0cqssHoTNU267KlrDuGZ1WYlxDStUtKUhOaJmh112/TZmHxxUfuJqPXSOm7tDyas0OSIQ==", - "dev": true, - "dependencies": { - "isexe": "^2.0.0" - }, - "bin": { - "which": "bin/which" - } - }, "node_modules/globals": { "version": "13.20.0", "dev": true, @@ -4013,35 +3393,11 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/hasown": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", - "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", - "dev": true, - "dependencies": { - "function-bind": "^1.1.2" - }, - "engines": { - "node": ">= 0.4" - } - }, "node_modules/heap": { "version": "0.2.7", "dev": true, "license": "MIT" }, - "node_modules/homedir-polyfill": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz", - "integrity": "sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA==", - "dev": true, - "dependencies": { - "parse-passwd": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/hosted-git-info": { "version": "6.1.1", "dev": true, @@ -4348,12 +3704,11 @@ } }, "node_modules/is-core-module": { - "version": "2.13.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", - "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", + "version": "2.11.0", "dev": true, + "license": "MIT", "dependencies": { - "hasown": "^2.0.0" + "has": "^1.0.3" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -4563,15 +3918,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/is-windows": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/is-windows/-/is-windows-1.0.2.tgz", - "integrity": "sha512-eXK1UInq2bPmjyX6e3VHIzMLobc4J94i4AWn+Hpq3OU5KkrRC96OAcR3PRJ/pGu6m8TRnBHP9dkXQVsT/COVIA==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -4660,9 +4006,8 @@ }, "node_modules/js-tokens": { "version": "4.0.0", - "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true + "dev": true, + "license": "MIT" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -4675,18 +4020,6 @@ "js-yaml": "bin/js-yaml.js" } }, - "node_modules/jsesc": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", - "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", - "dev": true, - "bin": { - "jsesc": "bin/jsesc" - }, - "engines": { - "node": ">=4" - } - }, "node_modules/json-diff": { "version": "0.9.0", "dev": true, @@ -4781,18 +4114,6 @@ "dev": true, "license": "MIT" }, - "node_modules/json5": { - "version": "2.2.3", - "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", - "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", - "dev": true, - "bin": { - "json5": "lib/cli.js" - }, - "engines": { - "node": ">=6" - } - }, "node_modules/jsonc-parser": { "version": "3.2.0", "license": "MIT" @@ -4872,7 +4193,6 @@ "version": "5.11.0", "resolved": "https://registry.npmjs.org/light-my-request/-/light-my-request-5.11.0.tgz", "integrity": "sha512-qkFCeloXCOMpmEdZ/MV91P8AT4fjwFXWaAFz3lUeStM8RcoM1ks4J/F8r1b3r6y/H4u3ACEJ1T+Gv5bopj7oDA==", - "dev": true, "dependencies": { "cookie": "^0.5.0", "process-warning": "^2.0.0", @@ -5248,24 +4568,6 @@ "version": "1.0.14", "license": "MIT" }, - "node_modules/magic-string": { - "version": "0.30.5", - "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.5.tgz", - "integrity": "sha512-7xlpfBaQaP/T6Vh8MO/EqXSW5En6INHEvEXQiuff7Gku0PWjU3uf6w/j9o7O+SpB5fOAkrI5HeoNgwjEO0pFsA==", - "dev": true, - "dependencies": { - "@jridgewell/sourcemap-codec": "^1.4.15" - }, - "engines": { - "node": ">=12" - } - }, - "node_modules/magic-string/node_modules/@jridgewell/sourcemap-codec": { - "version": "1.4.15", - "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.4.15.tgz", - "integrity": "sha512-eF2rxCRulEKXHTRiDrDy6erMYWqNw4LPdQ8UQA4huuxaQsVeRPFl2oM8oDGxMFhJUWZf9McpLtJasDDZb/Bpeg==", - "dev": true - }, "node_modules/make-dir": { "version": "3.1.0", "dev": true, @@ -5521,25 +4823,6 @@ "multicast-dns": "cli.js" } }, - "node_modules/multimatch": { - "version": "5.0.0", - "resolved": "https://registry.npmjs.org/multimatch/-/multimatch-5.0.0.tgz", - "integrity": "sha512-ypMKuglUrZUD99Tk2bUQ+xNQj43lPEfAeX2o9cTteAmShXy2VHDJpuwu1o0xqoKCt9jLVAvwyFKdLTPXKAfJyA==", - "dev": true, - "dependencies": { - "@types/minimatch": "^3.0.3", - "array-differ": "^3.0.0", - "array-union": "^2.1.0", - "arrify": "^2.0.1", - "minimatch": "^3.0.4" - }, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/mutexify": { "version": "1.4.0", "license": "MIT", @@ -5577,24 +4860,6 @@ "url": "https://github.com/chalk/chalk?sponsor=1" } }, - "node_modules/nanoid": { - "version": "3.3.7", - "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.7.tgz", - "integrity": "sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "bin": { - "nanoid": "bin/nanoid.cjs" - }, - "engines": { - "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" - } - }, "node_modules/napi-build-utils": { "version": "1.0.2", "license": "MIT" @@ -5981,7 +5246,6 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", "integrity": "sha512-0eJJY6hXLGf1udHwfNftBqH+g73EU4B504nZeKpz1sYRKafAghwxEJunB2O7rDZkL4PGfsMVnTXZ2EjibbqcsA==", - "dev": true, "engines": { "node": ">=14.0.0" } @@ -6197,15 +5461,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/parse-passwd": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz", - "integrity": "sha512-1Y1A//QUXEZK7YKz+rD9WydcE1+EuPr6ZBgKecAB8tmoW6UFv0NREVJe1p+jRxtThkcbbKkfwIbWJe/IeE6m2Q==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/patch-package": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/patch-package/-/patch-package-8.0.0.tgz", @@ -6351,12 +5606,6 @@ "node": ">=8" } }, - "node_modules/picocolors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.0.0.tgz", - "integrity": "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==", - "dev": true - }, "node_modules/picomatch": { "version": "2.3.1", "license": "MIT", @@ -6390,7 +5639,6 @@ "version": "8.16.1", "resolved": "https://registry.npmjs.org/pino/-/pino-8.16.1.tgz", "integrity": "sha512-3bKsVhBmgPjGV9pyn4fO/8RtoVDR8ssW1ev819FsRXlRNgW8gR/9Kx+gCK4UPWd4JjrRDLWpzd/pb1AyWm3MGA==", - "dev": true, "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", @@ -6412,7 +5660,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", - "dev": true, "dependencies": { "readable-stream": "^4.0.0", "split2": "^4.0.0" @@ -6422,7 +5669,6 @@ "version": "6.0.3", "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "dev": true, "funding": [ { "type": "github", @@ -6446,7 +5692,6 @@ "version": "4.4.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==", - "dev": true, "dependencies": { "abort-controller": "^3.0.0", "buffer": "^6.0.3", @@ -6461,45 +5706,7 @@ "node_modules/pino-std-serializers": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", - "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==", - "dev": true - }, - "node_modules/please-upgrade-node": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz", - "integrity": "sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg==", - "dev": true, - "dependencies": { - "semver-compare": "^1.0.0" - } - }, - "node_modules/postcss": { - "version": "8.4.31", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.31.tgz", - "integrity": "sha512-PS08Iboia9mts/2ygV3eLpY5ghnUcfLV/EXTOW1E2qYxJKGGBUtNjN76FYHnMs36RmARn41bC0AZmn+rR0OVpQ==", - "dev": true, - "funding": [ - { - "type": "opencollective", - "url": "https://opencollective.com/postcss/" - }, - { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/postcss" - }, - { - "type": "github", - "url": "https://github.com/sponsors/ai" - } - ], - "dependencies": { - "nanoid": "^3.3.6", - "picocolors": "^1.0.0", - "source-map-js": "^1.0.2" - }, - "engines": { - "node": "^10 || ^12 || >=14" - } + "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" }, "node_modules/prebuild-install": { "version": "7.1.1", @@ -6559,14 +5766,12 @@ "version": "0.11.10", "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", - "dev": true, "engines": { "node": ">= 0.6.0" } }, "node_modules/process-warning": { "version": "2.2.0", - "dev": true, "license": "MIT" }, "node_modules/protobufjs": { @@ -6617,7 +5822,6 @@ }, "node_modules/proxy-addr": { "version": "2.0.7", - "dev": true, "license": "MIT", "dependencies": { "forwarded": "0.2.0", @@ -6629,7 +5833,6 @@ }, "node_modules/proxy-addr/node_modules/ipaddr.js": { "version": "1.9.1", - "dev": true, "license": "MIT", "engines": { "node": ">= 0.10" @@ -6676,8 +5879,7 @@ "node_modules/quick-format-unescaped": { "version": "4.0.4", "resolved": "https://registry.npmjs.org/quick-format-unescaped/-/quick-format-unescaped-4.0.4.tgz", - "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==", - "dev": true + "integrity": "sha512-tYC1Q1hgyRuHgloV/YXs2w15unPVh8qfu/qCTfhTYamaw7fyhumKa2yGpdSo87vY32rIclj+4fWYQXUMs9EHvg==" }, "node_modules/quick-lru": { "version": "6.1.1", @@ -6970,18 +6172,6 @@ "node": ">= 6" } }, - "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, "node_modules/ready-resource": { "version": "1.0.0", "license": "MIT" @@ -6990,7 +6180,6 @@ "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", - "dev": true, "engines": { "node": ">= 12.13.0" } @@ -7048,19 +6237,12 @@ "node": ">=0.10.0" } }, - "node_modules/require-package-name": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/require-package-name/-/require-package-name-2.0.1.tgz", - "integrity": "sha512-uuoJ1hU/k6M0779t3VMVIYpb2VMJk05cehCaABFhXaibcbvfgR8wKiozLjVFSzJPmQMRqIcO0HMyTFqfV09V6Q==", - "dev": true - }, "node_modules/resolve": { - "version": "1.22.8", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", - "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", + "version": "1.22.1", "dev": true, + "license": "MIT", "dependencies": { - "is-core-module": "^2.13.0", + "is-core-module": "^2.9.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, @@ -7071,19 +6253,6 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/resolve-dir": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/resolve-dir/-/resolve-dir-1.0.1.tgz", - "integrity": "sha512-R7uiTjECzvOsWSfdM0QKFNBVFcK27aHOUwdvK53BcW8zqnGdYp0Fbj82cy54+2A4P2tFM22J5kRfe1R+lM/1yg==", - "dev": true, - "dependencies": { - "expand-tilde": "^2.0.0", - "global-modules": "^1.0.0" - }, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/resolve-from": { "version": "4.0.0", "dev": true, @@ -7144,14 +6313,12 @@ "version": "0.2.2", "resolved": "https://registry.npmjs.org/ret/-/ret-0.2.2.tgz", "integrity": "sha512-M0b3YWQs7R3Z917WRQy1HHA7Ba7D8hvZg6UE5mLykJxQVE2ju0IXbGlaHPPlkY+WN7wFP+wUMXmBFA0aV6vYGQ==", - "dev": true, "engines": { "node": ">=4" } }, "node_modules/reusify": { "version": "1.0.4", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -7160,7 +6327,6 @@ }, "node_modules/rfdc": { "version": "1.3.0", - "dev": true, "license": "MIT" }, "node_modules/rimraf": { @@ -7323,7 +6489,6 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/safe-regex2/-/safe-regex2-2.0.0.tgz", "integrity": "sha512-PaUSFsUaNNuKwkBijoAPHAK6/eM6VirvyPWlZ7BAQy4D+hCvh4B6lIG+nPdhbFfIbP+gTGBcrdsOaUs0F+ZBOQ==", - "dev": true, "dependencies": { "ret": "~0.2.0" } @@ -7332,7 +6497,6 @@ "version": "2.4.3", "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.4.3.tgz", "integrity": "sha512-e2bDA2WJT0wxseVd4lsDP4+3ONX6HpMXQa1ZhFQ7SU+GjvORCmShbCMltrtIDfkYhVHrOcPtj+KhmDBdPdZD1g==", - "dev": true, "engines": { "node": ">=10" } @@ -7352,7 +6516,6 @@ }, "node_modules/secure-json-parse": { "version": "2.7.0", - "dev": true, "license": "BSD-3-Clause" }, "node_modules/semver": { @@ -7363,15 +6526,8 @@ "semver": "bin/semver.js" } }, - "node_modules/semver-compare": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", - "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", - "dev": true - }, "node_modules/set-cookie-parser": { "version": "2.6.0", - "dev": true, "license": "MIT" }, "node_modules/sha256-universal": { @@ -7653,7 +6809,6 @@ "version": "3.7.0", "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.7.0.tgz", "integrity": "sha512-IudtNvSqA/ObjN97tfgNmOKyDOs4dNcg4cUUsHDebqsgb8wGBBwb31LIgShNO8fye0dFI52X1+tFoKKI6Rq1Gg==", - "dev": true, "dependencies": { "atomic-sleep": "^1.0.0" } @@ -7665,15 +6820,6 @@ "node": ">=0.10.0" } }, - "node_modules/source-map-js": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.0.2.tgz", - "integrity": "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/source-map-support": { "version": "0.5.21", "dev": true, @@ -7715,7 +6861,6 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", - "dev": true, "engines": { "node": ">= 10.x" } @@ -8042,7 +7187,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz", "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==", - "dev": true, "dependencies": { "real-require": "^0.2.0" } @@ -8100,15 +7244,6 @@ "node": ">=0.6.0" } }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "dev": true, - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "license": "MIT", @@ -8123,7 +7258,6 @@ "version": "3.3.0", "resolved": "https://registry.npmjs.org/toad-cache/-/toad-cache-3.3.0.tgz", "integrity": "sha512-3oDzcogWGHZdkwrHyvJVpPjA7oNzY6ENOV3PsWJY9XYPZ6INo94Yd47s5may1U+nleBPwDhrRiTPMIvKaa3MQg==", - "dev": true, "engines": { "node": ">=12" } @@ -8209,9 +7343,9 @@ } }, "node_modules/type-fest": { - "version": "4.8.0", - "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.8.0.tgz", - "integrity": "sha512-rIY1yHlQhXNRfRyUNnpBr9pr1qxCHSN80hNNHINWQvpgvrVnu3uoi20+mkRfSD1vud6fsA2VLU8AENZhj5jGCQ==", + "version": "4.8.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.8.1.tgz", + "integrity": "sha512-ShaaYnjf+0etG8W/FumARKMjjIToy/haCaTjN2dvcewOSoNqCQzdgG7m2JVOlM5qndGTHjkvsrWZs+k/2Z7E0Q==", "engines": { "node": ">=16" }, @@ -8576,9 +7710,8 @@ "license": "ISC" }, "node_modules/xache": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/xache/-/xache-1.2.0.tgz", - "integrity": "sha512-AiWCGTrnsh//rrbJt8DLbDkDW8eLp1Ktkq0nTWzpE+FKCY35oeqLjtz+LNb4abMnjfTgL0ZBaSwzhgzan1ocEw==" + "version": "1.1.0", + "license": "MIT" }, "node_modules/xsalsa20": { "version": "1.2.0", @@ -8596,15 +7729,6 @@ "version": "4.0.0", "license": "ISC" }, - "node_modules/yaml": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.2.tgz", - "integrity": "sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==", - "dev": true, - "engines": { - "node": ">= 6" - } - }, "node_modules/yargs": { "version": "16.2.0", "dev": true, From 5d7c251adf4ccbabf5d09a025d977a2b499a44cb Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Wed, 22 Nov 2023 08:42:28 +0900 Subject: [PATCH 65/69] ignore debug type errors --- src/logger.js | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/logger.js b/src/logger.js index 0c82fbc41..80de7b9db 100644 --- a/src/logger.js +++ b/src/logger.js @@ -24,20 +24,26 @@ createDebug.formatters.k = function (v) { * @param {import('./sync/sync-state.js').State} v * @this {any} */ createDebug.formatters.X = function (v) { - const mapped = mapObject(v, (k, v) => [ - k, - mapObject(v, (k, v) => { - if (k === 'remoteStates') - return [k, mapObject(v, (k, v) => [k.slice(0, 7), v])] - return [k, v] - }), - ]) - return util.inspect(mapped, { - colors: true, - depth: 10, - compact: 6, - breakLength: 90, - }) + try { + const mapped = mapObject(v, (k, v) => [ + k, + // @ts-ignore - type checks here don't get us anything + mapObject(v, (k, v) => { + if (k === 'remoteStates') + // @ts-ignore - type checks here don't get us anything + return [k, mapObject(v, (k, v) => [k.slice(0, 7), v])] + return [k, v] + }), + ]) + return util.inspect(mapped, { + colors: true, + depth: 10, + compact: 6, + breakLength: 90, + }) + } catch (e) { + return `[ERROR: $(e.message)]` + } } const counts = new Map() From a2f8a1b8bd5f55939d7d50f4a0753e3009005ae1 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Wed, 22 Nov 2023 08:53:06 +0900 Subject: [PATCH 66/69] fixes for review comments --- src/local-peers.js | 2 +- src/mapeo-manager.js | 2 +- src/mapeo-project.js | 6 +++++- src/sync/namespace-sync-state.js | 2 +- test-e2e/manager-invite.js | 2 +- 5 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/local-peers.js b/src/local-peers.js index ad032d5ef..fe3df44e6 100644 --- a/src/local-peers.js +++ b/src/local-peers.js @@ -207,7 +207,7 @@ class Peer { * @property {(peers: PeerInfo[]) => void} peers Emitted whenever the connection status of peers changes. An array of peerInfo objects with a peer id and the peer connection status * @property {(peer: PeerInfoConnected) => void} peer-add Emitted when a new peer is connected * @property {(peerId: string, invite: InviteWithKeys) => void} invite Emitted when an invite is received - * @property {(discoveryKey: Buffer, protomux: Protomux) => void} discovery-key Emitted when a new hypercore is replicated (by a peer) to a peer replication stream (passed as the second parameter) + * @property {(discoveryKey: Buffer, protomux: Protomux) => void} discovery-key Emitted when a new hypercore is replicated (by a peer) to a peer protomux instance (passed as the second parameter) */ /** @extends {TypedEmitter} */ diff --git a/src/mapeo-manager.js b/src/mapeo-manager.js index 053fe32e6..97672d2ea 100644 --- a/src/mapeo-manager.js +++ b/src/mapeo-manager.js @@ -534,7 +534,7 @@ export class MapeoManager extends TypedEmitter { * happening, this will never timeout, but if more than timeoutMs passes * without any sync activity, then this will resolve `false` e.g. data has not * synced - * @returns + * @returns {Promise} */ async #waitForInitialSync(project, { timeoutMs = 5000 } = {}) { await project.ready() diff --git a/src/mapeo-project.js b/src/mapeo-project.js index 9c759e857..48836c6cb 100644 --- a/src/mapeo-project.js +++ b/src/mapeo-project.js @@ -478,7 +478,11 @@ export class MapeoProject { const replicationStream = this.#coreManager.creatorCore.replicate(stream, { // @ts-ignore - hypercore types do not currently include this option ondiscoverykey: async (discoveryKey) => { - this.#syncApi[kHandleDiscoveryKey](discoveryKey, replicationStream) + const protomux = + /** @type {import('protomux')} */ ( + replicationStream.noiseStream.userData + ) + this.#syncApi[kHandleDiscoveryKey](discoveryKey, protomux) }, }) return replicationStream diff --git a/src/sync/namespace-sync-state.js b/src/sync/namespace-sync-state.js index 81934e3a9..ff2dec969 100644 --- a/src/sync/namespace-sync-state.js +++ b/src/sync/namespace-sync-state.js @@ -86,9 +86,9 @@ export class NamespaceSyncState { * @param {Buffer} coreKey */ #addCore(core, coreKey) { - this.#coreCount++ const discoveryId = discoveryKey(coreKey).toString('hex') this.#getCoreState(discoveryId).attachCore(core) + this.#coreCount++ } /** diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index 2500e1c62..d1bca57f1 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -112,7 +112,7 @@ test('chain of invites', async (t) => { await disconnectPeers(managers) }) -// Needs fix to inviteApi to check capabilities before sending invite +// TODO: Needs fix to inviteApi to check capabilities before sending invite skip("member can't invite", async (t) => { const managers = await createManagers(3) const [creator, member, joiner] = managers From cd89ec47326acca0cb1c2c856b9bee47ac330e8a Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Wed, 22 Nov 2023 08:59:19 +0900 Subject: [PATCH 67/69] move utils-new into utils --- test-e2e/manager-invite.js | 2 +- test-e2e/members.js | 2 +- test-e2e/project-crud.js | 4 +- test-e2e/sync.js | 2 +- test-e2e/utils-new.js | 295 ------------------------------------- test-e2e/utils.js | 295 +++++++++++++++++++++++++++++++++++++ 6 files changed, 300 insertions(+), 300 deletions(-) delete mode 100644 test-e2e/utils-new.js diff --git a/test-e2e/manager-invite.js b/test-e2e/manager-invite.js index d1bca57f1..83750410b 100644 --- a/test-e2e/manager-invite.js +++ b/test-e2e/manager-invite.js @@ -7,7 +7,7 @@ import { createManagers, disconnectPeers, waitForPeers, -} from './utils-new.js' +} from './utils.js' test('member invite accepted', async (t) => { const [creator, joiner] = await createManagers(2) diff --git a/test-e2e/members.js b/test-e2e/members.js index 23998c805..2934a531a 100644 --- a/test-e2e/members.js +++ b/test-e2e/members.js @@ -14,7 +14,7 @@ import { disconnectPeers, invite, waitForPeers, -} from './utils-new.js' +} from './utils.js' test('getting yourself after creating project', async (t) => { const [manager] = await createManagers(1) diff --git a/test-e2e/project-crud.js b/test-e2e/project-crud.js index a56ad9282..bb495f371 100644 --- a/test-e2e/project-crud.js +++ b/test-e2e/project-crud.js @@ -4,8 +4,8 @@ import { KeyManager } from '@mapeo/crypto' import { valueOf } from '../src/utils.js' import { MapeoManager } from '../src/mapeo-manager.js' import RAM from 'random-access-memory' -import { stripUndef } from './utils-new.js' -import { round } from './utils-new.js' +import { stripUndef } from './utils.js' +import { round } from './utils.js' /** @satisfies {Array} */ const fixtures = [ diff --git a/test-e2e/sync.js b/test-e2e/sync.js index beaee3e62..20abb7e58 100644 --- a/test-e2e/sync.js +++ b/test-e2e/sync.js @@ -6,7 +6,7 @@ import { seedDatabases, sortById, waitForSync, -} from './utils-new.js' +} from './utils.js' import { kCoreManager } from '../src/mapeo-project.js' import { getKeys } from '../tests/helpers/core-manager.js' import { NAMESPACES } from '../src/core-manager/index.js' diff --git a/test-e2e/utils-new.js b/test-e2e/utils-new.js deleted file mode 100644 index ed2f82587..000000000 --- a/test-e2e/utils-new.js +++ /dev/null @@ -1,295 +0,0 @@ -// @ts-check -import sodium from 'sodium-universal' -import RAM from 'random-access-memory' - -import { MapeoManager } from '../src/index.js' -import { kManagerReplicate, kRPC } from '../src/mapeo-manager.js' -import { MEMBER_ROLE_ID } from '../src/capabilities.js' -import { once } from 'node:events' -import { generate } from '@mapeo/mock-data' -import { valueOf } from '../src/utils.js' -import { randomInt } from 'node:crypto' - -/** - * @param {readonly MapeoManager[]} managers - */ -export async function disconnectPeers(managers) { - return Promise.all( - managers.map(async (manager) => { - return manager.stopLocalPeerDiscovery({ force: true }) - }) - ) -} - -/** - * @param {readonly MapeoManager[]} managers - */ -export function connectPeers(managers, { discovery = true } = {}) { - if (discovery) { - for (const manager of managers) { - manager.startLocalPeerDiscovery() - } - return function destroy() { - return disconnectPeers(managers) - } - } else { - /** @type {import('../src/types.js').ReplicationStream[]} */ - const replicationStreams = [] - for (let i = 0; i < managers.length; i++) { - for (let j = i + 1; j < managers.length; j++) { - const r1 = managers[i][kManagerReplicate](true) - const r2 = managers[j][kManagerReplicate](false) - replicationStreams.push(r1, r2) - r1.pipe(r2).pipe(r1) - } - } - return function destroy() { - const promises = [] - for (const stream of replicationStreams) { - promises.push( - /** @type {Promise} */ - ( - new Promise((res) => { - stream.on('close', res) - stream.destroy() - }) - ) - ) - } - return Promise.all(promises) - } - } -} - -/** - * Invite mapeo clients to a project - * - * @param {{ - * invitor: MapeoManager, - * projectId: string, - * invitees: MapeoManager[], - * roleId?: import('../src/capabilities.js').RoleId, - * reject?: boolean - * }} opts - */ -export async function invite({ - invitor, - projectId, - invitees, - roleId = MEMBER_ROLE_ID, - reject = false, -}) { - const invitorProject = await invitor.getProject(projectId) - const promises = [] - - for (const invitee of invitees) { - promises.push( - invitorProject.$member.invite(invitee.deviceId, { - roleId, - }) - ) - promises.push( - once(invitee.invite, 'invite-received').then(([invite]) => { - return reject - ? invitee.invite.reject(invite.projectId) - : invitee.invite.accept(invite.projectId) - }) - ) - } - - await Promise.allSettled(promises) -} - -/** - * Waits for all manager instances to be connected to each other - * - * @param {readonly MapeoManager[]} managers - */ -export async function waitForPeers(managers) { - const peerCounts = managers.map((manager) => { - return manager[kRPC].peers.filter(({ status }) => status === 'connected') - .length - }) - const expectedCount = managers.length - 1 - return new Promise((res) => { - if (peerCounts.every((v) => v === expectedCount)) { - return res(null) - } - for (const [idx, manager] of managers.entries()) { - manager.on('local-peers', function onPeers(peers) { - const connectedPeerCount = peers.filter( - ({ status }) => status === 'connected' - ).length - peerCounts[idx] = connectedPeerCount - if (connectedPeerCount === expectedCount) { - manager.off('local-peers', onPeers) - } - if (peerCounts.every((v) => v === expectedCount)) { - res(null) - } - }) - } - }) -} - -/** - * Create `count` manager instances. Each instance has a deterministic identity - * keypair so device IDs should be consistent between tests. - * - * @template {number} T - * @param {T} count - * @returns {Promise>} - */ -export async function createManagers(count) { - // @ts-ignore - return Promise.all( - Array(count) - .fill(null) - .map(async (_, i) => { - const name = 'device' + i - const manager = createManager(name) - await manager.setDeviceInfo({ name }) - return manager - }) - ) -} - -/** @param {string} [seed] */ -export function createManager(seed) { - return new MapeoManager({ - rootKey: getRootKey(seed), - dbFolder: ':memory:', - coreStorage: () => new RAM(), - }) -} - -/** @param {string} [seed] */ -function getRootKey(seed) { - const key = Buffer.allocUnsafe(16) - if (!seed) { - sodium.randombytes_buf(key) - } else { - const seedBuf = Buffer.alloc(32) - sodium.crypto_generichash(seedBuf, Buffer.from(seed)) - sodium.randombytes_buf_deterministic(key, seedBuf) - } - return key -} -/** - * Remove undefined properties from an object, to allow deep comparison - * @param {object} obj - */ -export function stripUndef(obj) { - return JSON.parse(JSON.stringify(obj)) -} -/** - * - * @param {number} value - * @param {number} decimalPlaces - */ -export function round(value, decimalPlaces) { - return Math.round(value * 10 ** decimalPlaces) / 10 ** decimalPlaces -} - -/** - * Unlike `mapeo.project.$sync.waitForSync` this also waits for the specified - * number of peers to connect. - * - * @param {import('../src/mapeo-project.js').MapeoProject} project - * @param {string[]} peerIds - * @param {'initial' | 'full'} [type] - */ -async function waitForProjectSync(project, peerIds, type = 'initial') { - const state = await project.$sync.getState() - if (hasPeerIds(state.auth.remoteStates, peerIds)) { - return project.$sync.waitForSync(type) - } - return new Promise((res) => { - project.$sync.on('sync-state', function onState(state) { - if (!hasPeerIds(state.auth.remoteStates, peerIds)) return - project.$sync.off('sync-state', onState) - res(project.$sync.waitForSync(type)) - }) - }) -} - -/** - * @param {Record} remoteStates - * @param {string[]} peerIds - * @returns - */ -function hasPeerIds(remoteStates, peerIds) { - for (const peerId of peerIds) { - if (!(peerId in remoteStates)) return false - } - return true -} - -/** - * Wait for all projects to connect and sync - * - * @param {import('../src/mapeo-project.js').MapeoProject[]} projects - * @param {'initial' | 'full'} [type] - */ -export function waitForSync(projects, type = 'initial') { - return Promise.all( - projects.map((project) => { - const peerIds = projects - .filter((p) => p !== project) - .map((p) => p.deviceId) - return waitForProjectSync(project, peerIds, type) - }) - ) -} - -/** - * @param {import('../src/mapeo-project.js').MapeoProject[]} projects - */ -export function seedDatabases(projects) { - return Promise.all(projects.map((p) => seedProjectDatabase(p))) -} - -const SCHEMAS_TO_SEED = /** @type {const} */ ([ - 'observation', - 'preset', - 'field', -]) - -/** - * @param {import('../src/mapeo-project.js').MapeoProject} project - * @returns {Promise>} - */ -async function seedProjectDatabase(project) { - const promises = [] - for (const schemaName of SCHEMAS_TO_SEED) { - const count = - schemaName === 'observation' ? randomInt(20, 100) : randomInt(0, 10) - let i = 0 - while (i++ < count) { - const value = valueOf(generate(schemaName)[0]) - promises.push( - // @ts-ignore - project[schemaName].create(value) - ) - } - } - return Promise.all(promises) -} - -/** - * @template {object} T - * @param {T[]} arr - * @param {keyof T} key - */ -export function sortBy(arr, key) { - return arr.sort(function (a, b) { - if (a[key] < b[key]) return -1 - if (a[key] > b[key]) return 1 - return 0 - }) -} - -/** @param {import('@mapeo/schema').MapeoDoc[]} docs */ -export function sortById(docs) { - return sortBy(docs, 'docId') -} diff --git a/test-e2e/utils.js b/test-e2e/utils.js index 04734e8f7..b60fbf739 100644 --- a/test-e2e/utils.js +++ b/test-e2e/utils.js @@ -1,3 +1,298 @@ +// @ts-check +import sodium from 'sodium-universal' +import RAM from 'random-access-memory' + +import { MapeoManager } from '../src/index.js' +import { kManagerReplicate, kRPC } from '../src/mapeo-manager.js' +import { MEMBER_ROLE_ID } from '../src/capabilities.js' +import { once } from 'node:events' +import { generate } from '@mapeo/mock-data' +import { valueOf } from '../src/utils.js' +import { randomInt } from 'node:crypto' + +/** + * @param {readonly MapeoManager[]} managers + */ +export async function disconnectPeers(managers) { + return Promise.all( + managers.map(async (manager) => { + return manager.stopLocalPeerDiscovery({ force: true }) + }) + ) +} + +/** + * @param {readonly MapeoManager[]} managers + */ +export function connectPeers(managers, { discovery = true } = {}) { + if (discovery) { + for (const manager of managers) { + manager.startLocalPeerDiscovery() + } + return function destroy() { + return disconnectPeers(managers) + } + } else { + /** @type {import('../src/types.js').ReplicationStream[]} */ + const replicationStreams = [] + for (let i = 0; i < managers.length; i++) { + for (let j = i + 1; j < managers.length; j++) { + const r1 = managers[i][kManagerReplicate](true) + const r2 = managers[j][kManagerReplicate](false) + replicationStreams.push(r1, r2) + r1.pipe(r2).pipe(r1) + } + } + return function destroy() { + const promises = [] + for (const stream of replicationStreams) { + promises.push( + /** @type {Promise} */ + ( + new Promise((res) => { + stream.on('close', res) + stream.destroy() + }) + ) + ) + } + return Promise.all(promises) + } + } +} + +/** + * Invite mapeo clients to a project + * + * @param {{ + * invitor: MapeoManager, + * projectId: string, + * invitees: MapeoManager[], + * roleId?: import('../src/capabilities.js').RoleId, + * reject?: boolean + * }} opts + */ +export async function invite({ + invitor, + projectId, + invitees, + roleId = MEMBER_ROLE_ID, + reject = false, +}) { + const invitorProject = await invitor.getProject(projectId) + const promises = [] + + for (const invitee of invitees) { + promises.push( + invitorProject.$member.invite(invitee.deviceId, { + roleId, + }) + ) + promises.push( + once(invitee.invite, 'invite-received').then(([invite]) => { + return reject + ? invitee.invite.reject(invite.projectId) + : invitee.invite.accept(invite.projectId) + }) + ) + } + + await Promise.allSettled(promises) +} + +/** + * Waits for all manager instances to be connected to each other + * + * @param {readonly MapeoManager[]} managers + */ +export async function waitForPeers(managers) { + const peerCounts = managers.map((manager) => { + return manager[kRPC].peers.filter(({ status }) => status === 'connected') + .length + }) + const expectedCount = managers.length - 1 + return new Promise((res) => { + if (peerCounts.every((v) => v === expectedCount)) { + return res(null) + } + for (const [idx, manager] of managers.entries()) { + manager.on('local-peers', function onPeers(peers) { + const connectedPeerCount = peers.filter( + ({ status }) => status === 'connected' + ).length + peerCounts[idx] = connectedPeerCount + if (connectedPeerCount === expectedCount) { + manager.off('local-peers', onPeers) + } + if (peerCounts.every((v) => v === expectedCount)) { + res(null) + } + }) + } + }) +} + +/** + * Create `count` manager instances. Each instance has a deterministic identity + * keypair so device IDs should be consistent between tests. + * + * @template {number} T + * @param {T} count + * @returns {Promise>} + */ +export async function createManagers(count) { + // @ts-ignore + return Promise.all( + Array(count) + .fill(null) + .map(async (_, i) => { + const name = 'device' + i + const manager = createManager(name) + await manager.setDeviceInfo({ name }) + return manager + }) + ) +} + +/** @param {string} [seed] */ +export function createManager(seed) { + return new MapeoManager({ + rootKey: getRootKey(seed), + dbFolder: ':memory:', + coreStorage: () => new RAM(), + }) +} + +/** @param {string} [seed] */ +function getRootKey(seed) { + const key = Buffer.allocUnsafe(16) + if (!seed) { + sodium.randombytes_buf(key) + } else { + const seedBuf = Buffer.alloc(32) + sodium.crypto_generichash(seedBuf, Buffer.from(seed)) + sodium.randombytes_buf_deterministic(key, seedBuf) + } + return key +} +/** + * Remove undefined properties from an object, to allow deep comparison + * @param {object} obj + */ +export function stripUndef(obj) { + return JSON.parse(JSON.stringify(obj)) +} +/** + * + * @param {number} value + * @param {number} decimalPlaces + */ +export function round(value, decimalPlaces) { + return Math.round(value * 10 ** decimalPlaces) / 10 ** decimalPlaces +} + +/** + * Unlike `mapeo.project.$sync.waitForSync` this also waits for the specified + * number of peers to connect. + * + * @param {import('../src/mapeo-project.js').MapeoProject} project + * @param {string[]} peerIds + * @param {'initial' | 'full'} [type] + */ +async function waitForProjectSync(project, peerIds, type = 'initial') { + const state = await project.$sync.getState() + if (hasPeerIds(state.auth.remoteStates, peerIds)) { + return project.$sync.waitForSync(type) + } + return new Promise((res) => { + project.$sync.on('sync-state', function onState(state) { + if (!hasPeerIds(state.auth.remoteStates, peerIds)) return + project.$sync.off('sync-state', onState) + res(project.$sync.waitForSync(type)) + }) + }) +} + +/** + * @param {Record} remoteStates + * @param {string[]} peerIds + * @returns + */ +function hasPeerIds(remoteStates, peerIds) { + for (const peerId of peerIds) { + if (!(peerId in remoteStates)) return false + } + return true +} + +/** + * Wait for all projects to connect and sync + * + * @param {import('../src/mapeo-project.js').MapeoProject[]} projects + * @param {'initial' | 'full'} [type] + */ +export function waitForSync(projects, type = 'initial') { + return Promise.all( + projects.map((project) => { + const peerIds = projects + .filter((p) => p !== project) + .map((p) => p.deviceId) + return waitForProjectSync(project, peerIds, type) + }) + ) +} + +/** + * @param {import('../src/mapeo-project.js').MapeoProject[]} projects + */ +export function seedDatabases(projects) { + return Promise.all(projects.map((p) => seedProjectDatabase(p))) +} + +const SCHEMAS_TO_SEED = /** @type {const} */ ([ + 'observation', + 'preset', + 'field', +]) + +/** + * @param {import('../src/mapeo-project.js').MapeoProject} project + * @returns {Promise>} + */ +async function seedProjectDatabase(project) { + const promises = [] + for (const schemaName of SCHEMAS_TO_SEED) { + const count = + schemaName === 'observation' ? randomInt(20, 100) : randomInt(0, 10) + let i = 0 + while (i++ < count) { + const value = valueOf(generate(schemaName)[0]) + promises.push( + // @ts-ignore + project[schemaName].create(value) + ) + } + } + return Promise.all(promises) +} + +/** + * @template {object} T + * @param {T[]} arr + * @param {keyof T} key + */ +export function sortBy(arr, key) { + return arr.sort(function (a, b) { + if (a[key] < b[key]) return -1 + if (a[key] > b[key]) return 1 + return 0 + }) +} + +/** @param {import('@mapeo/schema').MapeoDoc[]} docs */ +export function sortById(docs) { + return sortBy(docs, 'docId') +} /** * Lazy way of removing fields with undefined values from an object * @param {unknown} object From 83ffe9295e0255351d6104936602b2006d618b41 Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 23 Nov 2023 12:11:45 +0900 Subject: [PATCH 68/69] Add debug info to test that sometimes fails --- tests/local-peers.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/local-peers.js b/tests/local-peers.js index 084b7e8ba..9b151c7e7 100644 --- a/tests/local-peers.js +++ b/tests/local-peers.js @@ -109,7 +109,7 @@ test('Send invite, duplicate connections', async (t) => { t.is(peers3.length, 1) t.ok( peers3[0].connectedAt > peers1[0].connectedAt, - 'later connected peer is not used' + `later connected peer is not used: ${peers3[0].connectedAt} ${peers1[0].connectedAt}` ) { From 4b0c71b5194ad15e162c428d364f89ded6200b6c Mon Sep 17 00:00:00 2001 From: Gregor MacLennan Date: Thu, 23 Nov 2023 12:33:04 +0900 Subject: [PATCH 69/69] Update package-lock.json version --- package-lock.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package-lock.json b/package-lock.json index 51ef6996c..34a791f94 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@mapeo/core", - "version": "9.0.0-alpha.1", + "version": "9.0.0-alpha.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@mapeo/core", - "version": "9.0.0-alpha.1", + "version": "9.0.0-alpha.2", "hasInstallScript": true, "license": "MIT", "dependencies": {