Flush count overflow issue mitigation

README.rst

@@ -1,3 +1,14 @@
+===============================================
+ElectrumX autocompact
+===============================================
+This little fork adds a shell script and makes some smaller changes to
+allow for automatic history compaction. This means every 15 months or 
+so (Bitcoin) the server will stop for a couple of hours and restart
+automatically. Without these changes you have to run compaction manually.  
+
+From the contrib files only the systemd service file has been prepared to
+use autocompact.  
+
 .. image:: https://api.cirrus-ci.com/github/spesmilo/electrumx.svg?branch=master
     :target: https://cirrus-ci.com/github/spesmilo/electrumx
 .. image:: https://coveralls.io/repos/github/spesmilo/electrumx/badge.svg

contrib/systemd/electrumx.service

@@ -4,11 +4,19 @@ After=network.target
 
 [Service]
 EnvironmentFile=/etc/electrumx.conf
-ExecStart=/usr/local/bin/electrumx_server
-ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop
+#ExecStart=/usr/local/bin/electrumx_server
+ExecStart=/home/electrumx/electrumx/electrumx_server_autocompact.sh
+#ExecStop=/usr/local/bin/electrumx_rpc -p 8000 stop
 User=electrumx
 LimitNOFILE=8192
 TimeoutStopSec=30min
+#CPUQuota=30%
+
+# CPU and IO depriorisation - default=100
+#CPUAccounting=true
+#CPUWeight=10
+#IOAccounting=true
+#IOWeight=10
 
 [Install]
 WantedBy=multi-user.target

docs/HOWTO.rst

@@ -237,6 +237,27 @@ See also `contrib/raspberrypi3/run_electrumx.sh`_ for an easy way to
 configure and launch electrumx.
 
 
+Flush Count Overflow
+====================
+
+After some months depending on blocktime electrumx will stop with this Error::
+
+  HistoryFlushCountOverflowException: History needs to be compacted now!
+
+With Bitcoin this will happen very roughly every 15 months. Before it happens
+there will be warnings in the log. The database then needs to be compacted
+using the electrumx_compact_history script supplied with your version of
+electrumx. It needs to be run with electrumx stopped and using the same
+environment variables. This can be done automatically by starting electrumx
+via ``electrumx_server_autocompact.sh`` or manually like this from an account
+with access to the database folder::
+
+  export $(cat /etc/electrumx.conf | grep -v "#" | xargs) && python3 ./electrumx_compact_history
+
+Compaction can take several hours. To postpone compaction for while you can
+increase `HISTORY_FLUSH_COUNT_MAX`.
+
+
 Sync Progress
 =============
 

electrumx/server/db.py

@@ -269,6 +269,9 @@ def flush_dbs(self, flush_data, flush_utxos, estimate_txs_remaining):
             self.logger.info(f'sync time: {formatted_time(self.wall_time)}  '
                              f'ETA: {formatted_time(eta)}')
 
+        # Now that everything is done check if history flush counter is high
+        self.check_history_flush_counter()
+
     def flush_fs(self, flush_data):
         '''Write headers, tx counts and block tx hashes to the filesystem.
 
@@ -312,6 +315,9 @@ def flush_fs(self, flush_data):
     def flush_history(self):
         self.history.flush()
 
+    def check_history_flush_counter(self):
+        self.history.check_flush_counter(self.env.history_flush_count_max)
+
     def flush_utxo_db(self, batch, flush_data: FlushData):
         '''Flush the cached DB writes and UTXO set to the batch.'''
         # Care is needed because the writes generated by flushing the

electrumx/server/env.py

@@ -76,6 +76,7 @@ def __init__(self, coin=None):
         self.blacklist_url = self.default('BLACKLIST_URL', self.coin.BLACKLIST_URL)
         self.cache_MB = self.integer('CACHE_MB', 1200)
         self.reorg_limit = self.integer('REORG_LIMIT', self.coin.REORG_LIMIT)
+        self.history_flush_count_max = self.integer('HISTORY_FLUSH_COUNT_MAX', 60000)
 
         # Server limits to help prevent DoS
 

electrumx/server/history.py

@@ -28,6 +28,10 @@
 FLUSHID_LEN = 2
 
 
+class HistoryFlushCountOverflowException(Exception):
+    pass
+
+
 class History:
 
     DB_VERSIONS = (0, 1)
@@ -175,6 +179,18 @@ def flush(self):
             self.logger.info(f'flushed history in {elapsed:.1f}s '
                              f'for {count:,d} addrs')
 
+    def check_flush_counter(self, history_flush_count_max):
+        # Warning
+        if not self.flush_count % 1000:  # 1000, 2000, 3000, ...
+            self.logger.info(f'History flush_count is at {self.flush_count:d} ' +
+                             f'of {history_flush_count_max:d}')
+            if self.flush_count >= history_flush_count_max - 10000:
+                self.logger.warning('History needs to be compacted soon! See HOWTO')
+
+        # Meaningful exception
+        if self.flush_count >= min(history_flush_count_max, 65535):
+            raise HistoryFlushCountOverflowException('History needs to be compacted now! See HOWTO')
+
     def backup(self, hashXs, tx_count):
         # Not certain this is needed, but it doesn't hurt
         self.flush_count += 1

electrumx_server

@@ -16,6 +16,7 @@ import sys
 from electrumx import Controller, Env
 from electrumx.lib.util import CompactFormatter, make_logger
 
+from electrumx.server.history import HistoryFlushCountOverflowException
 
 def main():
     '''Set up logging and run the server.'''
@@ -33,6 +34,9 @@ def main():
         logger.setLevel(env.log_level)
         controller = Controller(env)
         asyncio.run(controller.run())
+    except HistoryFlushCountOverflowException:
+        logger.exception('ElectrumX server terminated with HistoryFlushCountOverflowException')
+        sys.exit(65)
     except Exception:
         logger.exception('ElectrumX server terminated abnormally')
     else:

electrumx_server_autocompact.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+while ./electrumx_server; exitcode=$? && test $exitcode -eq 65;
+do
+    echo Attempting automatic electrumx_compact_history.
+    ./electrumx_compact_history
+    # exit on compaction failure
+    if test $? -ne 0
+    then
+        exit 2
+    fi
+done
+exit $exitcode