Account lastReference cache, now with Block support.

As this changes how lastReferences are checked and updated,
this is not suitable for rolling into current chain without a
"feature trigger", or chain restart!

Added unit tests.

src/main/java/org/qortal/account/AccountRefCache.java

@@ -3,20 +3,66 @@ package org.qortal.account;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Map.Entry;
-import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
 
 import org.qortal.data.account.AccountData;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.utils.Pair;
 
+/**
+ * Account lastReference caching
+ * <p>
+ * When checking an account's lastReference, the value returned should be the
+ * most recent value set after processing the most recent block.
+ * <p>
+ * However, when processing a batch of transactions, e.g. during block processing or validation,
+ * each transaction needs to check, and maybe update, multiple accounts' lastReference values.
+ * <p>
+ * Because the intermediate updates would affect future checks, we set up a cache of that
+ * maintains a consistent value for fetching lastReference, but also tracks the latest new
+ * value, without the overhead of repository calls.
+ * <p>
+ * Thus, when batch transaction processing is finished, only the latest new lastReference values
+ * can be committed to the repository, via {@link AccountRefCache#commit()}.
+ * <p>
+ * Getting and setting lastReferences values are done the usual way via
+ * {@link Account#getLastReference()} and {@link Account#setLastReference(byte[])} which call
+ * package-visibility methods in <tt>AccountRefCache</tt>.
+ * <p>
+ * If {@link Account#getLastReference()} or {@link Account#setLastReference(byte[])} are called
+ * outside of caching then lastReference values are fetched/set directly from/to the repository.
+ * <p>
+ * <tt>AccountRefCache</tt> implements <tt>AutoCloseable</tt> for (typical) use in a try-with-resources block.
+ *
+ * @see Account#getLastReference()
+ * @see Account#setLastReference(byte[])
+ * @see org.qortal.block.Block#process()
+ */
 public class AccountRefCache implements AutoCloseable {
 
-	private static final Map<Repository, RefCache> CACHE = new HashMap<Repository, RefCache>();
+	private static final Map<Repository, RefCache> CACHE = new HashMap<>();
 
 	private static class RefCache {
-		private final Map<String, byte[]> getLastReferenceValues = new HashMap<String, byte[]>();
-		private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<String, Pair<byte[], byte[]>>();
+		private final Map<String, byte[]> getLastReferenceValues = new HashMap<>();
+		private final Map<String, Pair<byte[], byte[]>> setLastReferenceValues = new HashMap<>();
+
+		/**
+		 * Function for merging publicKey from new data with old publicKey from map.
+		 * <p>
+		 * Last reference is <tt>A</tt> element in pair.<br>
+		 * Public key is <tt>B</tt> element in pair.
+		 */
+		private static final BinaryOperator<Pair<byte[], byte[]>> mergePublicKey = (oldPair,  newPair) -> {
+			// If passed new pair contains non-null publicKey, then we use that one in preference.
+			if (newPair.getB() == null)
+				// Otherwise, inherit publicKey from old map value.
+				newPair.setB(oldPair.getB());
+
+			// We always use new lastReference from new pair.
+			return newPair;
+		};
+
 
 		public byte[] getLastReference(Repository repository, String address) throws DataException {
 			synchronized (this.getLastReferenceValues) {
@@ -36,13 +82,11 @@ public class AccountRefCache implements AutoCloseable {
 		}
 
 		public void setLastReference(AccountData accountData) {
-			BiFunction<String, Pair<byte[], byte[]>, Pair<byte[], byte[]>> mergePublicKey = (key, oldPair) -> {
-				byte[] mergedPublicKey = accountData.getPublicKey() != null ? accountData.getPublicKey() : oldPair.getB();
-				return new Pair<>(accountData.getReference(), mergedPublicKey);
-			};
+			// We're only interested in lastReference and publicKey
+			Pair<byte[], byte[]> newPair = new Pair<>(accountData.getReference(), accountData.getPublicKey());
 
 			synchronized (this.setLastReferenceValues) {
-				setLastReferenceValues.computeIfPresent(accountData.getAddress(), mergePublicKey);
+				setLastReferenceValues.merge(accountData.getAddress(), newPair, mergePublicKey);
 			}
 		}
 
@@ -53,6 +97,12 @@ public class AccountRefCache implements AutoCloseable {
 
 	private Repository repository;
 
+	/**
+	 * Constructs a new account reference cache, unique to passed <tt>repository</tt> handle.
+	 * 
+	 * @param repository
+	 * @throws IllegalStateException if a cache already exists for <tt>repository</tt>
+	 */
 	public AccountRefCache(Repository repository) {
 		RefCache refCache = new RefCache();
 
@@ -64,9 +114,17 @@ public class AccountRefCache implements AutoCloseable {
 		this.repository = repository;
 	}
 
+	/**
+	 * Save all cached setLastReference account-reference values into repository.
+	 * <p>
+	 * Closes cache to prevent any future setLastReference() attempts post-commit.
+	 * 
+	 * @throws DataException
+	 */
 	public void commit() throws DataException {
 		RefCache refCache;
 
+		// Also duplicated in close(), this prevents future setLastReference() attempts post-commit.
 		synchronized (CACHE) {
 			refCache = CACHE.remove(this.repository);
 		}
@@ -89,12 +147,29 @@ public class AccountRefCache implements AutoCloseable {
 	}
 
 	@Override
-	public void close() throws Exception {
+	public void close() {
 		synchronized (CACHE) {
 			CACHE.remove(this.repository);
 		}
 	}
 
+	/**
+	 * Returns lastReference value for account.
+	 * <p>
+	 * If cache is not in effect for passed <tt>repository</tt> handle,
+	 * then this method fetches lastReference directly from repository.
+	 * <p>
+	 * If cache <i>is</i> in effect, then this method returns cached
+	 * lastReference, which is <b>not</b> affected by calls to
+	 * <tt>setLastReference</tt>.
+	 * <p>
+	 * Typically called by corresponding method in Account class.
+	 * 
+	 * @param repository
+	 * @param address account's address
+	 * @return account's lastReference, or null if account unknown, or lastReference not set
+	 * @throws DataException
+	 */
 	/*package*/ static byte[] getLastReference(Repository repository, String address) throws DataException {
 		RefCache refCache;
 
@@ -108,6 +183,22 @@ public class AccountRefCache implements AutoCloseable {
 		return refCache.getLastReference(repository, address);
 	}
 
+	/**
+	 * Sets lastReference value for account.
+	 * <p>
+	 * If cache is not in effect for passed <tt>repository</tt> handle,
+	 * then this method sets lastReference directly in repository.
+	 * <p>
+	 * If cache <i>is</i> in effect, then this method caches the new
+	 * lastReference, which is <b>not</b> returned by calls to
+	 * <tt>getLastReference</tt>.
+	 * <p>
+	 * Typically called by corresponding method in Account class.
+	 * 
+	 * @param repository
+	 * @param accountData
+	 * @throws DataException
+	 */
 	/*package*/ static void setLastReference(Repository repository, AccountData accountData) throws DataException {
 		RefCache refCache;
 
@@ -115,8 +206,10 @@ public class AccountRefCache implements AutoCloseable {
 			refCache = CACHE.get(repository);
 		}
 
-		if (refCache == null)
+		if (refCache == null) {
 			repository.getAccountRepository().setLastReference(accountData);
+			return;
+		}
 
 		refCache.setLastReference(accountData);
 	}

src/main/java/org/qortal/block/Block.java

@@ -18,6 +18,7 @@ import java.util.stream.Collectors;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.account.Account;
+import org.qortal.account.AccountRefCache;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.account.PublicKeyAccount;
 import org.qortal.asset.Asset;
@@ -1015,7 +1016,9 @@ public class Block {
 
 	/** Returns whether block's transactions are valid. */
 	private ValidationResult areTransactionsValid() throws DataException {
-		try {
+		// We're about to (test-)process a batch of transactions,
+		// so create an account reference cache so get/set correct last-references.
+		try (AccountRefCache accountRefCache = new AccountRefCache(repository)) {
 			// Create repository savepoint here so we can rollback to it after testing transactions
 			repository.setSavepoint();
 
@@ -1229,14 +1232,21 @@ public class Block {
 			rewardTransactionFees();
 		}
 
-		// Process transactions (we'll link them to this block after saving the block itself)
-		processTransactions();
+		// We're about to (test-)process a batch of transactions,
+		// so create an account reference cache so get/set correct last-references.
+		try (AccountRefCache accountRefCache = new AccountRefCache(this.repository)) {
+			// Process transactions (we'll link them to this block after saving the block itself)
+			processTransactions();
 
-		// Group-approval transactions
-		processGroupApprovalTransactions();
+			// Group-approval transactions
+			processGroupApprovalTransactions();
 
-		// Process AT fees and save AT states into repository
-		processAtFeesAndStates();
+			// Process AT fees and save AT states into repository
+			processAtFeesAndStates();
+
+			// Commit new accounts' last-reference changes
+			accountRefCache.commit();
+		}
 
 		// Link block into blockchain by fetching signature of highest block and setting that as our reference
 		BlockData latestBlockData = this.repository.getBlockRepository().fromHeight(blockchainHeight);