Rework of AT state trimming and pruning, in order to more reliably track the "latest" AT states.

This should fix an edge case where AT states data was pruned/trimmed but it was then later required in consensus. The older state was deleted because it was replaced by a new "latest" state in a brand new block. But once the new "latest" state was orphaned from the block, the old "latest" state was then required again.

This works around the problem by excluding very recent blocks in the latest AT states data, so that it is unaffected by real-time sync activity.

The trade off is that we could end up retaining more AT states than needed, so a secondary cleanup process may need to run at some time in the future to remove these. But it should only be a minimal amount of data, and can be cleaned up with a single query. This would have been happening to a certain degree already.

src/main/java/org/qortal/controller/repository/AtStatesPruner.java

@@ -39,9 +39,10 @@ public class AtStatesPruner implements Runnable {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int pruneStartHeight = repository.getATRepository().getAtPruneHeight();
+			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();
@@ -91,7 +92,8 @@ public class AtStatesPruner implements Runnable {
 					if (upperPrunableHeight > upperBatchHeight) {
 						pruneStartHeight = upperBatchHeight;
 						repository.getATRepository().setAtPruneHeight(pruneStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
+						maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+						repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 						repository.saveChanges();
 
 						final int finalPruneStartHeight = pruneStartHeight;

src/main/java/org/qortal/controller/repository/AtStatesTrimmer.java

@@ -26,9 +26,10 @@ public class AtStatesTrimmer implements Runnable {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			int trimStartHeight = repository.getATRepository().getAtTrimHeight();
+			int maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
 
 			repository.discardChanges();
-			repository.getATRepository().rebuildLatestAtStates();
+			repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 
 			while (!Controller.isStopping()) {
 				repository.discardChanges();
@@ -69,7 +70,8 @@ public class AtStatesTrimmer implements Runnable {
 					if (upperTrimmableHeight > upperBatchHeight) {
 						trimStartHeight = upperBatchHeight;
 						repository.getATRepository().setAtTrimHeight(trimStartHeight);
-						repository.getATRepository().rebuildLatestAtStates();
+						maxLatestAtStatesHeight = PruneManager.getMaxHeightForLatestAtStates(repository);
+						repository.getATRepository().rebuildLatestAtStates(maxLatestAtStatesHeight);
 						repository.saveChanges();
 
 						final int finalTrimStartHeight = trimStartHeight;

src/main/java/org/qortal/controller/repository/PruneManager.java

@@ -157,4 +157,18 @@ public class PruneManager {
         return (height < latestUnprunedHeight);
     }
 
+    /**
+     * When rebuilding the latest AT states, we need to specify a maxHeight, so that we aren't tracking
+     * very recent AT states that could potentially be orphaned. This method ensures that AT states
+     * are given a sufficient number of blocks to confirm before being tracked as a latest AT state.
+     */
+    public static int getMaxHeightForLatestAtStates(Repository repository) throws DataException {
+        // Get current chain height, and subtract a certain number of "confirmation" blocks
+        // This is to ensure we are basing our latest AT states data on confirmed blocks -
+        // ones that won't be orphaned in any normal circumstances
+        final int confirmationBlocks = 250;
+        final int chainHeight = repository.getBlockRepository().getBlockchainHeight();
+        return chainHeight - confirmationBlocks;
+    }
+
 }

src/main/java/org/qortal/repository/ATRepository.java

@@ -119,7 +119,7 @@ public interface ATRepository {
 	 * <p>
 	 * NOTE: performs implicit <tt>repository.saveChanges()</tt>.
 	 */
-	public void rebuildLatestAtStates() throws DataException;
+	public void rebuildLatestAtStates(int maxHeight) throws DataException;
 
 
 	/** Returns height of first trimmable AT state. */

src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java

@@ -603,7 +603,7 @@ public class HSQLDBATRepository implements ATRepository {
 
 
 	@Override
-	public void rebuildLatestAtStates() throws DataException {
+	public void rebuildLatestAtStates(int maxHeight) throws DataException {
 		// latestATStatesLock is to prevent concurrent updates on LatestATStates
 		// that could result in one process using a partial or empty dataset
 		// because it was in the process of being rebuilt by another thread
@@ -624,11 +624,12 @@ public class HSQLDBATRepository implements ATRepository {
 					+ "CROSS JOIN LATERAL("
 					+ "SELECT height FROM ATStates "
 					+ "WHERE ATStates.AT_address = ATs.AT_address "
+					+ "AND height <= ?"
 					+ "ORDER BY AT_address DESC, height DESC LIMIT 1"
 					+ ") "
 					+ ")";
 			try {
-				this.repository.executeCheckedUpdate(insertSql);
+				this.repository.executeCheckedUpdate(insertSql, maxHeight);
 			} catch (SQLException e) {
 				repository.examineException(e);
 				throw new DataException("Unable to populate temporary latest AT states cache in repository", e);

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabasePruning.java

@@ -99,7 +99,7 @@ public class HSQLDBDatabasePruning {
 
         // It's essential that we rebuild the latest AT states here, as we are using this data in the next query.
         // Failing to do this will result in important AT states being deleted, rendering the database unusable.
-        repository.getATRepository().rebuildLatestAtStates();
+        repository.getATRepository().rebuildLatestAtStates(endHeight);
 
 
         // Loop through all the LatestATStates and copy them to the new table