From 10ac07531657b664aec7c720071d46bcda701ce3 Mon Sep 17 00:00:00 2001 From: Christian Bundy Date: Tue, 17 Sep 2019 15:06:48 -0700 Subject: [PATCH 1/2] Add write every 10K messages --- inject.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/inject.js b/inject.js index 0e52785..85f5a04 100644 --- a/inject.js +++ b/inject.js @@ -143,14 +143,24 @@ return function (version, reduce, map, codec, initial) { }, createSink: function (cb) { closeStream = cb; + let count = 0 return Drain(function (data) { var _data = map(data.value, data.seq) if(_data != null) value.set(reduce(value.value, _data, data.seq)) since.set(data.seq) notify(_data) - //if we are now in sync with the log, maybe write. - if(since.value === log.since.value) + + // If we are now in sync with the log, write. + const inSyncWithLog = since.value === log.since.value + + // Alternatively, write every 10,000 messages. + const isTenThousand = count % 10000 + + if(inSyncWithLog || isTenThousand) { write() + } + + count += 1 }, cb) }, destroy: function (cb) { From 2ddf4bc952e44d3004db4b1a27bd1f5054926756 Mon Sep 17 00:00:00 2001 From: Daan Wynen Date: Sun, 10 Jan 2021 18:29:21 +0100 Subject: [PATCH 2/2] Make write interval a parameter. Problem: as Dominic pointed out, flumedb is general purpose, so even if one write every 10K messages is a good compromise for ssb, it might be problematic for other projects. Solution: don't do periodic writes by default, but do allow passing a parameter to the module to set them up. Also I guess the periodic counter condition was inverted. :P --- inject.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/inject.js b/inject.js index 85f5a04..e08d9ea 100644 --- a/inject.js +++ b/inject.js @@ -31,7 +31,7 @@ function isFunction (f) { function id (e) { return e } module.exports = function (Store) { -return function (version, reduce, map, codec, initial) { +return function (version, reduce, map, codec, initial, writeInterval = null) { var opts if(isObject(reduce)) { opts = reduce @@ -154,9 +154,9 @@ return function (version, reduce, map, codec, initial) { const inSyncWithLog = since.value === log.since.value // Alternatively, write every 10,000 messages. - const isTenThousand = count % 10000 + const periodicWrite = writeInterval && count % writeInterval == 0 - if(inSyncWithLog || isTenThousand) { + if(inSyncWithLog || periodicWrite) { write() }