From 20e3d9bea334477527b41d35be935495b48314f3 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:21:08 +0000
Subject: [PATCH 01/17] Update db.py: make env available to history.py

---
 electrumx/server/db.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/electrumx/server/db.py b/electrumx/server/db.py
index fae1885d6..bc2e40366 100644
--- a/electrumx/server/db.py
+++ b/electrumx/server/db.py
@@ -92,7 +92,7 @@ def __init__(self, env: 'Env'):
         os.chdir(env.db_dir)
 
         self.db_class = db_class(self.env.db_engine)
-        self.history = History()
+        self.history = History(self.env)
 
         # Key: b'u' + address_hashX + txout_idx + tx_num
         # Value: the UTXO value as a 64-bit unsigned integer (in satoshis)

From 880edd5e11030e169a26b7f245a1c622dd743b6a Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:22:23 +0000
Subject: [PATCH 02/17] Update env.py: add HISTORY_FLUSH_COUNT_MAX

---
 electrumx/server/env.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/electrumx/server/env.py b/electrumx/server/env.py
index 12735a1e0..89237830f 100644
--- a/electrumx/server/env.py
+++ b/electrumx/server/env.py
@@ -76,7 +76,8 @@ def __init__(self, coin=None):
         self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
         self.cache_MB = self.integer('CACHE_MB', 1200)
         self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
-
+        self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)  # deactivated if > 65535
+        
         # Server limits to help prevent DoS
 
         self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)

From d3bdcec735b34e227bcb0b04fd9dae97508e6aaa Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:27:17 +0000
Subject: [PATCH 03/17] Update history.py: add flush count exception, warnings

---
 electrumx/server/history.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/electrumx/server/history.py b/electrumx/server/history.py
index dc7dd8d54..df5305c37 100644
--- a/electrumx/server/history.py
+++ b/electrumx/server/history.py
@@ -28,14 +28,19 @@
 FLUSHID_LEN = 2
 
 
+class HistoryFlushCountOverflowException(Exception):
+    pass
+
+
 class History:
 
     DB_VERSIONS = (0, 1)
 
     db: Optional['Storage']
 
-    def __init__(self):
+    def __init__(self, env):
         self.logger = util.class_logger(__name__, self.__class__.__name__)
+        self.env = env
         # For history compaction
         self.max_hist_row_entries = 12500
         self.unflushed = defaultdict(bytearray)
@@ -157,6 +162,15 @@ def assert_flushed(self):
     def flush(self):
         start_time = time.monotonic()
         self.flush_count += 1
+
+        if not self.flush_count % 1000:  # 1000, 2000, 3000, ...
+            self.logger.info(f'History flush_count is at {self.flush_count:d} ' +
+                             f'of {self.env.history_flush_count_max:d}')
+            if self.flush_count >= self.env.history_flush_count_max - 10000:
+                self.logger.warning('History needs to be compacted soon! See HOWTO')
+        if self.flush_count > self.env.history_flush_count_max:
+            raise HistoryFlushCountOverflowException('History needs to be compacted! See HOWTO')
+
         flush_id = pack_be_uint16(self.flush_count)
         unflushed = self.unflushed
 

From cea1e78d06f611957916be46fb4589040d218854 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Thu, 27 Apr 2023 10:34:27 +0000
Subject: [PATCH 04/17] Update HOWTO.rst add Flush Count Overflow paragraph

---
 docs/HOWTO.rst | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst
index 4ec30fb19..0258a2c2b 100644
--- a/docs/HOWTO.rst
+++ b/docs/HOWTO.rst
@@ -237,6 +237,26 @@ See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to
 configure and launch electrumx.
 
 
+Flush Count Overflow
+====================
+
+After some months depending on blocktime electrumx will stop with this Error::
+
+  HistoryFlushCountOverflowException: History needs to be compacted!
+
+With Bitcoin this will happen very roughly every 15 months. Before it happens
+there will be warnings in the log. The database then needs to be compacted
+using the electrumx_compact_history script supplied with your version of
+electrumx. It needs to be run with electrumx stopped and using the same
+environment variables. This can be done like this from an account with access
+to the database folder::
+
+  export $(cat /etc/electrumx.conf | grep -v "#" | xargs) && python3 ./electrumx_compact_history
+
+Compaction can take several hours. To postpone compaction for while you can
+increase `HISTORY_FLUSH_COUNT_MAX`.
+
+
 Sync Progress
 =============
 

From 46b88e755dddf5e7f87ab45cd926e56ca5344cca Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 2 May 2023 10:22:39 +0000
Subject: [PATCH 05/17] Update history.py: move OverflowException to the end of
 the flush function

---
 electrumx/server/history.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/electrumx/server/history.py b/electrumx/server/history.py
index df5305c37..743ffffa8 100644
--- a/electrumx/server/history.py
+++ b/electrumx/server/history.py
@@ -168,8 +168,6 @@ def flush(self):
                              f'of {self.env.history_flush_count_max:d}')
             if self.flush_count >= self.env.history_flush_count_max - 10000:
                 self.logger.warning('History needs to be compacted soon! See HOWTO')
-        if self.flush_count > self.env.history_flush_count_max:
-            raise HistoryFlushCountOverflowException('History needs to be compacted! See HOWTO')
 
         flush_id = pack_be_uint16(self.flush_count)
         unflushed = self.unflushed
@@ -189,6 +187,9 @@ def flush(self):
             self.logger.info(f'flushed history in {elapsed:.1f}s '
                              f'for {count:,d} addrs')
 
+        if self.flush_count >= min(self.env.history_flush_count_max, 65535):
+            raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
+
     def backup(self, hashXs, tx_count):
         # Not certain this is needed, but it doesn't hurt
         self.flush_count += 1

From f4c3128d7419eb4fe9e0b0c3dba34e4645e6acde Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 9 May 2023 08:44:35 +0000
Subject: [PATCH 06/17] Update db.py: refactoring

---
 electrumx/server/db.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/electrumx/server/db.py b/electrumx/server/db.py
index bc2e40366..2a99ba57e 100644
--- a/electrumx/server/db.py
+++ b/electrumx/server/db.py
@@ -92,7 +92,7 @@ def __init__(self, env: 'Env'):
         os.chdir(env.db_dir)
 
         self.db_class = db_class(self.env.db_engine)
-        self.history = History(self.env)
+        self.history = History()
 
         # Key: b'u' + address_hashX + txout_idx + tx_num
         # Value: the UTXO value as a 64-bit unsigned integer (in satoshis)
@@ -269,6 +269,9 @@ def flush_dbs(self, flush_data, flush_utxos, estimate_txs_remaining):
             self.logger.info(f'sync time: {formatted_time(self.wall_time)}  '
                              f'ETA: {formatted_time(eta)}')
 
+        # Now that everything is done check if history flush counter is high
+        self.check_history_flush_counter()
+
     def flush_fs(self, flush_data):
         '''Write headers, tx counts and block tx hashes to the filesystem.
 
@@ -312,6 +315,9 @@ def flush_fs(self, flush_data):
     def flush_history(self):
         self.history.flush()
 
+    def check_history_flush_counter(self):
+        self.history.check_flush_counter(self.env.history_flush_count_max)
+
     def flush_utxo_db(self, batch, flush_data: FlushData):
         '''Flush the cached DB writes and UTXO set to the batch.'''
         # Care is needed because the writes generated by flushing the

From c7777399d07705e31919509a80c504186ddab522 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 9 May 2023 08:49:08 +0000
Subject: [PATCH 07/17] Update history.py: refactoring to make final flush
 count

---
 electrumx/server/history.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/electrumx/server/history.py b/electrumx/server/history.py
index 743ffffa8..46f87983d 100644
--- a/electrumx/server/history.py
+++ b/electrumx/server/history.py
@@ -38,9 +38,8 @@ class History:
 
     db: Optional['Storage']
 
-    def __init__(self, env):
+    def __init__(self):
         self.logger = util.class_logger(__name__, self.__class__.__name__)
-        self.env = env
         # For history compaction
         self.max_hist_row_entries = 12500
         self.unflushed = defaultdict(bytearray)
@@ -162,13 +161,6 @@ def assert_flushed(self):
     def flush(self):
         start_time = time.monotonic()
         self.flush_count += 1
-
-        if not self.flush_count % 1000:  # 1000, 2000, 3000, ...
-            self.logger.info(f'History flush_count is at {self.flush_count:d} ' +
-                             f'of {self.env.history_flush_count_max:d}')
-            if self.flush_count >= self.env.history_flush_count_max - 10000:
-                self.logger.warning('History needs to be compacted soon! See HOWTO')
-
         flush_id = pack_be_uint16(self.flush_count)
         unflushed = self.unflushed
 
@@ -190,7 +182,19 @@ def flush(self):
         if self.flush_count >= min(self.env.history_flush_count_max, 65535):
             raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
 
-    def backup(self, hashXs, tx_count):
+    def check_flush_counter(self, history_flush_count_max):
+        # Warning
+        if not self.flush_count % 1000:  # 1000, 2000, 3000, ...
+            self.logger.info(f'History flush_count is at {self.flush_count:d} ' +
+                             f'of {history_flush_count_max:d}')
+            if self.flush_count >= history_flush_count_max - 10000:
+                self.logger.warning('History needs to be compacted soon! See HOWTO')
+
+        # Meaningful exception
+        if self.flush_count >= min(history_flush_count_max, 65535):
+            raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
+
+def backup(self, hashXs, tx_count):
         # Not certain this is needed, but it doesn't hurt
         self.flush_count += 1
         nremoves = 0

From 70977b7cabbe3abc33a5bc4cf8692ae2737b8261 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 9 May 2023 09:50:34 +0000
Subject: [PATCH 08/17] Update env.py spaces...

---
 electrumx/server/env.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/electrumx/server/env.py b/electrumx/server/env.py
index 89237830f..61506ab7a 100644
--- a/electrumx/server/env.py
+++ b/electrumx/server/env.py
@@ -77,7 +77,7 @@ def __init__(self, coin=None):
         self.cache_MB = self.integer('CACHE_MB', 1200)
         self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
         self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)  # deactivated if > 65535
-        
+
         # Server limits to help prevent DoS
 
         self.max_send = self.integer('MAX_SEND', self.coin.DEFAULT_MAX_SEND)

From 10bec1ccf490269696094b5fd82072aea72acf57 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 9 May 2023 09:52:52 +0000
Subject: [PATCH 09/17] Update history.py cleanup

---
 electrumx/server/history.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/electrumx/server/history.py b/electrumx/server/history.py
index 46f87983d..b3ed709fa 100644
--- a/electrumx/server/history.py
+++ b/electrumx/server/history.py
@@ -179,9 +179,6 @@ def flush(self):
             self.logger.info(f'flushed history in {elapsed:.1f}s '
                              f'for {count:,d} addrs')
 
-        if self.flush_count >= min(self.env.history_flush_count_max, 65535):
-            raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
-
     def check_flush_counter(self, history_flush_count_max):
         # Warning
         if not self.flush_count % 1000:  # 1000, 2000, 3000, ...

From ab102dcf36dec703936cb2715148b792b04b14fb Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Tue, 9 May 2023 11:11:33 +0000
Subject: [PATCH 10/17] Update history.py fix indentation

---
 electrumx/server/history.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/electrumx/server/history.py b/electrumx/server/history.py
index b3ed709fa..e4bfc3f5c 100644
--- a/electrumx/server/history.py
+++ b/electrumx/server/history.py
@@ -191,7 +191,7 @@ def check_flush_counter(self, history_flush_count_max):
         if self.flush_count >= min(history_flush_count_max, 65535):
             raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
 
-def backup(self, hashXs, tx_count):
+    def backup(self, hashXs, tx_count):
         # Not certain this is needed, but it doesn't hurt
         self.flush_count += 1
         nremoves = 0

From 3f1a095b25c5723871eaed1f65a0aff22661ad87 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Wed, 10 May 2023 07:13:28 +0000
Subject: [PATCH 11/17] Update env.py remove comment because of pystyle

---
 electrumx/server/env.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/electrumx/server/env.py b/electrumx/server/env.py
index 61506ab7a..34b3ec454 100644
--- a/electrumx/server/env.py
+++ b/electrumx/server/env.py
@@ -76,7 +76,7 @@ def __init__(self, coin=None):
         self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
         self.cache_MB = self.integer('CACHE_MB', 1200)
         self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
-        self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)  # deactivated if > 65535
+        self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)
 
         # Server limits to help prevent DoS
 

From 5e9dd2f704eb7cff1992e82219ae502904374b59 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 10:27:39 +0000
Subject: [PATCH 12/17] Update HOWTO.rst

---
 docs/HOWTO.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst
index 0258a2c2b..5e89912a7 100644
--- a/docs/HOWTO.rst
+++ b/docs/HOWTO.rst
@@ -242,7 +242,7 @@ Flush Count Overflow
 
 After some months depending on blocktime electrumx will stop with this Error::
 
-  HistoryFlushCountOverflowException: History needs to be compacted!
+  HistoryFlushCountOverflowException: History needs to be compacted now!
 
 With Bitcoin this will happen very roughly every 15 months. Before it happens
 there will be warnings in the log. The database then needs to be compacted

From 890e7746492a1c3e1804cacbc4302b5796f82dbb Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 10:33:41 +0000
Subject: [PATCH 13/17] Create electrumx_server_autocompact.sh

---
 electrumx_server_autocompact.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)
 create mode 100644 electrumx_server_autocompact.sh

diff --git a/electrumx_server_autocompact.sh b/electrumx_server_autocompact.sh
new file mode 100644
index 000000000..11285755e
--- /dev/null
+++ b/electrumx_server_autocompact.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+while ./electrumx_server; exitcode=$? && test $exitcode -eq 65;
+do
+    echo Attempting automatic electrumx_compact_history.
+    ./electrumx_compact_history
+    # exit on compaction failure
+    if test $? -ne 0
+    then
+        exit 2
+    fi
+done
+exit $exitcode

From 0ddeff140b644fc8becd2eb84fd5356c60691a4b Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 10:36:30 +0000
Subject: [PATCH 14/17] Update electrumx_server: exit with special code on
 flush_count overflow exception

---
 electrumx_server | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/electrumx_server b/electrumx_server
index efbe5d4d3..186219e90 100755
--- a/electrumx_server
+++ b/electrumx_server
@@ -16,6 +16,7 @@ import sys
 from electrumx import Controller, Env
 from electrumx.lib.util import CompactFormatter, make_logger
 
+from electrumx.server.history import HistoryFlushCountOverflowException
 
 def main():
     '''Set up logging and run the server.'''
@@ -33,6 +34,9 @@ def main():
         logger.setLevel(env.log_level)
         controller = Controller(env)
         asyncio.run(controller.run())
+    except HistoryFlushCountOverflowException:
+        logger.exception('ElectrumX server terminated with HistoryFlushCountOverflowException')
+        sys.exit(65)
     except Exception:
         logger.exception('ElectrumX server terminated abnormally')
     else:

From 3b502141ecb406ea1965118c9c40f9e5cd391721 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 11:07:46 +0000
Subject: [PATCH 15/17] Update electrumx.service switch to autocompact

---
 contrib/systemd/electrumx.service | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/contrib/systemd/electrumx.service b/contrib/systemd/electrumx.service
index 04d5b2043..8ce9d314c 100644
--- a/contrib/systemd/electrumx.service
+++ b/contrib/systemd/electrumx.service
@@ -4,11 +4,19 @@ After=network.target
 
 [Service]
 EnvironmentFile=/etc/electrumx.conf
-ExecStart=/usr/local/bin/electrumx_server
-ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop
+#ExecStart=/usr/local/bin/electrumx_server
+ExecStart=/home/electrumx/electrumx/electrumx_server_autocompact.sh
+#ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop
 User=electrumx
 LimitNOFILE=8192
 TimeoutStopSec=30min
+#CPUQuota=30%
+
+# CPU and IO depriorisation - default=100
+#CPUAccounting=true
+#CPUWeight=10
+#IOAccounting=true
+#IOWeight=10
 
 [Install]
 WantedBy=multi-user.target

From 30556c885a5b110d2992a004bc4516fd71aa56a0 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 11:19:22 +0000
Subject: [PATCH 16/17] Update HOWTO.rst add autocompact info

---
 docs/HOWTO.rst | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/HOWTO.rst b/docs/HOWTO.rst
index 5e89912a7..44dc83aac 100644
--- a/docs/HOWTO.rst
+++ b/docs/HOWTO.rst
@@ -248,8 +248,9 @@ With Bitcoin this will happen very roughly every 15 months. Before it happens
 there will be warnings in the log. The database then needs to be compacted
 using the electrumx_compact_history script supplied with your version of
 electrumx. It needs to be run with electrumx stopped and using the same
-environment variables. This can be done like this from an account with access
-to the database folder::
+environment variables. This can be done automatically by starting electrumx
+via ``electrumx_server_autocompact.sh`` or manually like this from an account
+with access to the database folder::
 
   export $(cat /etc/electrumx.conf | grep -v "#" | xargs) && python3 ./electrumx_compact_history
 

From 82d66c5d7f33a30977cb86e934d32e7a68040576 Mon Sep 17 00:00:00 2001
From: ghoober <131757532+ghoober@users.noreply.github.com>
Date: Mon, 15 May 2023 11:31:04 +0000
Subject: [PATCH 17/17] Update README.rst

---
 README.rst | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/README.rst b/README.rst
index 6d973e240..dc4169ca0 100644
--- a/README.rst
+++ b/README.rst
@@ -1,3 +1,14 @@
+===============================================
+ElectrumX autocompact
+===============================================
+This little fork adds a shell script and makes some smaller changes to
+allow for automatic history compaction. This means every 15 months or 
+so (Bitcoin) the server will stop for a couple of hours and restart
+automatically. Without these changes you have to run compaction manually.  
+  
+From the contrib files only the systemd service file has been prepared to
+use autocompact.  
+  
 .. image:: https://api.cirrus-ci.com/github/spesmilo/electrumx.svg?branch=master
     :target: https://cirrus-ci.com/github/spesmilo/electrumx
 .. image:: https://coveralls.io/repos/github/spesmilo/electrumx/badge.svg