From d93e9d570f4e2a5ea68b1cffc4e3a82fd35e7e6d Mon Sep 17 00:00:00 2001
From: catbref <misc-github@talk2dom.com>
Date: Fri, 11 Sep 2020 15:57:12 +0100
Subject: [PATCH] Trimming old block online accounts signatures

There's still an existing issue where log entries like this appear:

  Unable to trim old online accounts signatures in repository

which is actually caused by:

  integrity constraint violation: unique constraint or index violation; SYS_PK_10092 table: BLOCKS

which seems to be a bug in the version of HSQLDB we use.
(Tested using synced-from-scratch DB).
It's not clear what the actual problem is at this point.

It might be possible to switch to v2.5.1 if our recent HSQLDB-related
commits have fixed/worked-around the OOM issues.

Move the inner method from BlockChain to Controller.
Remove blockchain lock as it's not needed because it's not an
HSQLDB "serialization failure" but constraint violation.

Trimming old online accounts signatures limited to batches of 1440
rows to reduce CPU and memory load.
---
 .../java/org/qortal/block/BlockChain.java     | 30 -------------------
 .../org/qortal/controller/Controller.java     | 19 +++++++++++-
 .../hsqldb/HSQLDBBlockRepository.java         |  5 +++-
 3 files changed, 22 insertions(+), 32 deletions(-)

diff --git a/src/main/java/org/qortal/block/BlockChain.java b/src/main/java/org/qortal/block/BlockChain.java
index 809e678b..8f6ddab7 100644
--- a/src/main/java/org/qortal/block/BlockChain.java
+++ b/src/main/java/org/qortal/block/BlockChain.java
@@ -32,7 +32,6 @@ import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
-import org.qortal.utils.NTP;
 import org.qortal.utils.StringLongMapXmlAdapter;
 
 /**
@@ -578,33 +577,4 @@ public class BlockChain {
 		}
 	}
 
-	public static void trimOldOnlineAccountsSignatures() {
-		final Long now = NTP.getTime();
-		if (now == null)
-			return;
-
-		ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
-		if (!blockchainLock.tryLock())
-			// Too busy to trim right now, try again later
-			return;
-
-		try {
-			try (final Repository repository = RepositoryManager.tryRepository()) {
-				if (repository == null)
-					return;
-
-				int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime());
-
-				if (numBlocksTrimmed > 0)
-					LOGGER.debug(String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : "")));
-
-				repository.saveChanges();
-			} catch (DataException e) {
-				LOGGER.warn(String.format("Repository issue trying to trim old online accounts signatures: %s", e.getMessage()));
-			}
-		} finally {
-			blockchainLock.unlock();
-		}
-	}
-
 }
diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java
index 50db9763..4e9c6e76 100644
--- a/src/main/java/org/qortal/controller/Controller.java
+++ b/src/main/java/org/qortal/controller/Controller.java
@@ -1407,7 +1407,24 @@ public class Controller extends Thread {
 		sendOurOnlineAccountsInfo();
 
 		// Trim blockchain by removing 'old' online accounts signatures
-		BlockChain.trimOldOnlineAccountsSignatures();
+		long upperMintedTimestamp = now - BlockChain.getInstance().getOnlineAccountSignaturesMaxLifetime();
+		trimOldOnlineAccountsSignatures(upperMintedTimestamp);
+	}
+
+	private void trimOldOnlineAccountsSignatures(long upperMintedTimestamp) {
+		try (final Repository repository = RepositoryManager.tryRepository()) {
+			if (repository == null)
+				return;
+
+			int numBlocksTrimmed = repository.getBlockRepository().trimOldOnlineAccountsSignatures(upperMintedTimestamp);
+
+			if (numBlocksTrimmed > 0)
+				LOGGER.debug(() -> String.format("Trimmed old online accounts signatures from %d block%s", numBlocksTrimmed, (numBlocksTrimmed != 1 ? "s" : "")));
+
+			repository.saveChanges();
+		} catch (DataException e) {
+			LOGGER.warn(String.format("Repository issue trying to trim old online accounts signatures: %s", e.getMessage()));
+		}
 	}
 
 	private void sendOurOnlineAccountsInfo() {
diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java
index 56f3377f..c9d9de50 100644
--- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java
+++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockRepository.java
@@ -465,11 +465,14 @@ public class HSQLDBBlockRepository implements BlockRepository {
 
 	@Override
 	public int trimOldOnlineAccountsSignatures(long timestamp) throws DataException {
-		String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL";
+		// We're often called so no need to trim all blocks in one go.
+		// Limit updates to reduce CPU and memory load.
+		String sql = "UPDATE Blocks set online_accounts_signatures = NULL WHERE minted_when < ? AND online_accounts_signatures IS NOT NULL LIMIT 1440";
 
 		try {
 			return this.repository.executeCheckedUpdate(sql, timestamp);
 		} catch (SQLException e) {
+			repository.examineException(e);
 			throw new DataException("Unable to trim old online accounts signatures in repository", e);
 		}
 	}