fix: wait for index idle before returning data

package.json

@@ -129,7 +129,7 @@
     "hyperswarm": "4.4.1",
     "magic-bytes.js": "^1.0.14",
     "map-obj": "^5.0.2",
-    "multi-core-indexer": "1.0.0-alpha.7",
+    "multi-core-indexer": "^1.0.0-alpha.9",
     "p-defer": "^4.0.0",
     "p-timeout": "^6.1.2",
     "patch-package": "^8.0.0",

src/core-ownership.js

@@ -65,7 +65,7 @@ export class CoreOwnership {
       expressions.push(eq(table[`${namespace}CoreId`], coreId))
     }
     // prettier-ignore
-    const result = this.#dataType[kSelect]()
+    const result = (await this.#dataType[kSelect]())
       .where(or.apply(null, expressions))
       .get()
     if (!result) {

src/datastore/index.js

@@ -84,6 +84,10 @@ export class DataStore extends TypedEmitter {
     })
   }
 
+  get indexer() {
+    return this.#coreIndexer
+  }
+
   get namespace() {
     return this.#namespace
   }

src/datatype/index.js

@@ -140,6 +140,7 @@ export class DataType {
    * @param {string} docId
    */
   async getByDocId(docId) {
+    await this.#dataStore.indexer.idle()
     const result = this.#sql.getByDocId.get({ docId })
     if (!result) throw new Error('Not found')
     return deNullify(result)
@@ -151,6 +152,7 @@ export class DataType {
   }
 
   async getMany() {
+    await this.#dataStore.indexer.idle()
     return this.#sql.getMany.all().map((doc) => deNullify(doc))
   }
 
@@ -161,6 +163,7 @@ export class DataType {
    * @param {T} value
    */
   async update(versionId, value) {
+    await this.#dataStore.indexer.idle()
     const links = Array.isArray(versionId) ? versionId : [versionId]
     const { docId, createdAt, createdBy } = await this.#validateLinks(links)
     /** @type {any} */
@@ -181,6 +184,7 @@ export class DataType {
    * @param {string | string[]} versionId
    */
   async delete(versionId) {
+    await this.#dataStore.indexer.idle()
     const links = Array.isArray(versionId) ? versionId : [versionId]
     const { docId, createdAt, createdBy } = await this.#validateLinks(links)
     /** @type {any} */
@@ -200,7 +204,8 @@ export class DataType {
   /**
    * @param {Parameters<import('drizzle-orm/better-sqlite3').BetterSQLite3Database['select']>[0]} fields
    */
-  [kSelect](fields) {
+  async [kSelect](fields) {
+    await this.#dataStore.indexer.idle()
     return this.#db.select(fields).from(this.#table)
   }
 

test-e2e/manager-basic.js

@@ -280,7 +280,9 @@ test('Consistent storage folders', async (t) => {
       },
       { waitForSync: false }
     )
-    await manager.getProject(projectId)
+    const project = await manager.getProject(projectId)
+    // awaiting this ensures that indexing is done, which means that indexer storage is created
+    await project.$getOwnCapabilities()
   }
 
   // @ts-ignore snapshot() is missing from typedefs

test-e2e/sync.js

@@ -18,8 +18,8 @@ import { BLOCKED_ROLE_ID, COORDINATOR_ROLE_ID } from '../src/capabilities.js'
 
 const SCHEMAS_INITIAL_SYNC = ['preset', 'field']
 
-test('Create and sync data', async function (t) {
-  const COUNT = 5
+test('Create and sync data', { timeout: 100_000 }, async function (t) {
+  const COUNT = 10
   const managers = await createManagers(COUNT, t)
   const [invitor, ...invitees] = managers
   const disconnect = connectPeers(managers, { discovery: false })
@@ -252,5 +252,9 @@ test('no sync capabilities === no namespaces sync apart from auth', async (t) =>
     t.alike(invitorState[ns].localState, inviteeState[ns].localState)
   }
 
+  // Temp fix until we have .close() method - waits for indexing idle to ensure
+  // we don't close storage in teardown while index is still being written.
+  await Promise.all(projects.map((p) => p.$getProjectSettings()))
+
   await disconnect1()
 })

test-e2e/utils.js

@@ -12,7 +12,7 @@ import { randomInt } from 'node:crypto'
 import { temporaryDirectory } from 'tempy'
 import fsPromises from 'node:fs/promises'
 
-const FAST_TESTS = true
+const FAST_TESTS = !!process.env.FAST_TESTS
 const projectMigrationsFolder = new URL('../drizzle/project', import.meta.url)
   .pathname
 const clientMigrationsFolder = new URL('../drizzle/client', import.meta.url)

tests/datastore.js

@@ -100,10 +100,6 @@ test('index events', async (t) => {
   await dataStore.write(obs)
   await idlePromise
   const expectedStates = [
-    {
-      current: 'idle',
-      remaining: 0,
-    },
     {
       current: 'indexing',
       remaining: 1,