Flush count overflow issue mitigation

docs/HOWTO.rst

@@ -237,6 +237,26 @@ See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to
 configure and launch electrumx.
 
 
+Flush Count Overflow
+====================
+
+After some months depending on blocktime electrumx will stop with this Error::
+
+  HistoryFlushCountOverflowException: History needs to be compacted!
+
+With Bitcoin this will happen very roughly every 15 months. Before it happens
+there will be warnings in the log. The database then needs to be compacted
+using the electrumx_compact_history script supplied with your version of
+electrumx. It needs to be run with electrumx stopped and using the same
+environment variables. This can be done like this from an account with access
+to the database folder::
+
+  export $(cat /etc/electrumx.conf | grep -v "#" | xargs) && python3 ./electrumx_compact_history
+
+Compaction can take several hours. To postpone compaction for while you can
+increase `HISTORY_FLUSH_COUNT_MAX`.
+
+
 Sync Progress
 =============
 

electrumx/server/db.py

@@ -92,7 +92,7 @@ def __init__(self, env: 'Env'):
         os.chdir(env.db_dir)
 
         self.db_class = db_class(self.env.db_engine)
-        self.history = History()
+        self.history = History(self.env)
 
         # Key: b'u' + address_hashX + txout_idx + tx_num
         # Value: the UTXO value as a 64-bit unsigned integer (in satoshis)

electrumx/server/env.py

@@ -76,7 +76,8 @@ def __init__(self, coin=None):
         self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
         self.cache_MB = self.integer('CACHE_MB', 1200)
         self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
-
+        self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)  # deactivated if > 65535
+
         # Server limits to help prevent DoS
 
         self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)

electrumx/server/history.py

@@ -28,14 +28,19 @@
 FLUSHID_LEN = 2
 
 
+class HistoryFlushCountOverflowException(Exception):
+    pass
+
+
 class History:
 
     DB_VERSIONS = (0, 1)
 
     db: Optional['Storage']
 
-    def __init__(self):
+    def __init__(self, env):
         self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.env = env
         # For history compaction
         self.max_hist_row_entries = 12500
         self.unflushed = defaultdict(bytearray)
@@ -157,6 +162,15 @@ def assert_flushed(self):
     def flush(self):
         start_time = time.monotonic()
         self.flush_count += 1
+
+        if not self.flush_count % 1000:  # 1000, 2000, 3000, ...
+            self.logger.info(f'History flush_count is at {self.flush_count:d} ' +
+                             f'of {self.env.history_flush_count_max:d}')
+            if self.flush_count >= self.env.history_flush_count_max - 10000:
+                self.logger.warning('History needs to be compacted soon! See HOWTO')
+        if self.flush_count > self.env.history_flush_count_max:
+            raise HistoryFlushCountOverflowException('History needs to be compacted! See HOWTO')
+
         flush_id = pack_be_uint16(self.flush_count)
         unflushed = self.unflushed