Merge branch 'master' into chain-weight-consensus

# Conflicts:
#	src/main/java/org/qortal/block/BlockChain.java
#	src/main/resources/blockchain.json
#	src/test/resources/test-chain-v2-founder-rewards.json
#	src/test/resources/test-chain-v2-leftover-reward.json
#	src/test/resources/test-chain-v2-minting.json
#	src/test/resources/test-chain-v2-qora-holder-extremes.json
#	src/test/resources/test-chain-v2-qora-holder.json
#	src/test/resources/test-chain-v2-reward-scaling.json
#	src/test/resources/test-chain-v2.json

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -547,7 +547,7 @@ public class AdminResource {
 			blockchainLock.lockInterruptibly();
 
 			try {
-				repository.exportNodeLocalData();
+				repository.exportNodeLocalData(true);
 				return "true";
 			} finally {
 				blockchainLock.unlock();
@@ -628,25 +628,9 @@ public class AdminResource {
 	public String checkpointRepository() {
 		Security.checkApiCallAllowed(request);
 
-		try (final Repository repository = RepositoryManager.getRepository()) {
-			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+		RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
 
-			blockchainLock.lockInterruptibly();
-
-			try {
-				repository.checkpoint(true);
-				repository.saveChanges();
-
-				return "true";
-			} finally {
-				blockchainLock.unlock();
-			}
-		} catch (InterruptedException e) {
-			// We couldn't lock blockchain to perform checkpoint
-			return "false";
-		} catch (DataException e) {
-			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
-		}
+		return "true";
 	}
 
 	@POST

src/main/java/org/qortal/block/Block.java

@@ -176,19 +176,26 @@ public class Block {
 		 * 
 		 *  @return account-level share "bin" from blockchain config, or null if founder / none found
 		 */
-		public AccountLevelShareBin getShareBin() {
+		public AccountLevelShareBin getShareBin(int blockHeight) {
 			if (this.isMinterFounder)
 				return null;
 
 			final int accountLevel = this.mintingAccountData.getLevel();
 			if (accountLevel <= 0)
-					return null;
+				return null; // level 0 isn't included in any share bins
 
-			final AccountLevelShareBin[] shareBinsByLevel = BlockChain.getInstance().getShareBinsByAccountLevel();
+			final BlockChain blockChain = BlockChain.getInstance();
+			final AccountLevelShareBin[] shareBinsByLevel = blockChain.getShareBinsByAccountLevel();
 			if (accountLevel > shareBinsByLevel.length)
 				return null;
 
-			return shareBinsByLevel[accountLevel];
+			if (blockHeight < blockChain.getShareBinFixHeight())
+				// Off-by-one bug still in effect
+				return shareBinsByLevel[accountLevel];
+
+			// level 1 stored at index 0, level 2 stored at index 1, etc.
+			return shareBinsByLevel[accountLevel-1];
+
 		}
 
 		public long distribute(long accountAmount, Map<String, Long> balanceChanges) {
@@ -357,7 +364,7 @@ public class Block {
 			System.arraycopy(onlineAccountData.getSignature(), 0, onlineAccountsSignatures, i * Transformer.SIGNATURE_LENGTH, Transformer.SIGNATURE_LENGTH);
 		}
 
-		byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(),
+		byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData,
 				minter.getPublicKey(), encodedOnlineAccounts));
 
 		// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
@@ -424,7 +431,7 @@ public class Block {
 		int version = this.blockData.getVersion();
 		byte[] reference = this.blockData.getReference();
 
-		byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData.getMinterSignature(),
+		byte[] minterSignature = minter.sign(BlockTransformer.getBytesForMinterSignature(parentBlockData,
 				minter.getPublicKey(), this.blockData.getEncodedOnlineAccounts()));
 
 		// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level
@@ -738,11 +745,7 @@ public class Block {
 		if (!(this.minter instanceof PrivateKeyAccount))
 			throw new IllegalStateException("Block's minter is not a PrivateKeyAccount - can't sign!");
 
-		try {
-			this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData)));
-		} catch (TransformationException e) {
-			throw new RuntimeException("Unable to calculate block's minter signature", e);
-		}
+		this.blockData.setMinterSignature(((PrivateKeyAccount) this.minter).sign(BlockTransformer.getBytesForMinterSignature(this.blockData)));
 	}
 
 	/**
@@ -1331,6 +1334,9 @@ public class Block {
 
 		// Give Controller our cached, valid online accounts data (if any) to help reduce CPU load for next block
 		Controller.getInstance().pushLatestBlocksOnlineAccounts(this.cachedValidOnlineAccounts);
+
+		// Log some debugging info relating to the block weight calculation
+		this.logDebugInfo();
 	}
 
 	protected void increaseAccountLevels() throws DataException {
@@ -1512,6 +1518,9 @@ public class Block {
 	public void orphan() throws DataException {
 		LOGGER.trace(() -> String.format("Orphaning block %d", this.blockData.getHeight()));
 
+		// Log some debugging info relating to the block weight calculation
+		this.logDebugInfo();
+
 		// Return AT fees and delete AT states from repository
 		orphanAtFeesAndStates();
 
@@ -1786,7 +1795,7 @@ public class Block {
 			// Find all accounts in share bin. getShareBin() returns null for minter accounts that are also founders, so they are effectively filtered out.
 			AccountLevelShareBin accountLevelShareBin = accountLevelShareBins.get(binIndex);
 			// Object reference compare is OK as all references are read-only from blockchain config.
-			List<ExpandedAccount> binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin() == accountLevelShareBin).collect(Collectors.toList());
+			List<ExpandedAccount> binnedAccounts = expandedAccounts.stream().filter(accountInfo -> accountInfo.getShareBin(this.blockData.getHeight()) == accountLevelShareBin).collect(Collectors.toList());
 
 			// No online accounts in this bin? Skip to next one
 			if (binnedAccounts.isEmpty())
@@ -1984,4 +1993,33 @@ public class Block {
 		this.repository.getAccountRepository().tidy();
 	}
 
+	private void logDebugInfo() {
+		try {
+			if (this.repository == null || this.getMinter() == null || this.getBlockData() == null)
+				return;
+
+			int minterLevel = Account.getRewardShareEffectiveMintingLevel(this.repository, this.getMinter().getPublicKey());
+
+			LOGGER.debug(String.format("======= BLOCK %d (%.8s) =======", this.getBlockData().getHeight(), Base58.encode(this.getSignature())));
+			LOGGER.debug(String.format("Timestamp: %d", this.getBlockData().getTimestamp()));
+			LOGGER.debug(String.format("Minter level: %d", minterLevel));
+			LOGGER.debug(String.format("Online accounts: %d", this.getBlockData().getOnlineAccountsCount()));
+
+			BlockSummaryData blockSummaryData = new BlockSummaryData(this.getBlockData());
+			if (this.getParent() == null || this.getParent().getSignature() == null || blockSummaryData == null)
+				return;
+
+			blockSummaryData.setMinterLevel(minterLevel);
+			BigInteger blockWeight = calcBlockWeight(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData);
+			BigInteger keyDistance = calcKeyDistance(this.getParent().getHeight(), this.getParent().getSignature(), blockSummaryData.getMinterPublicKey(), blockSummaryData.getMinterLevel());
+			NumberFormat formatter = new DecimalFormat("0.###E0");
+
+			LOGGER.debug(String.format("Key distance: %s", formatter.format(keyDistance)));
+			LOGGER.debug(String.format("Weight: %s", formatter.format(blockWeight)));
+
+		} catch (DataException e) {
+			LOGGER.info(() -> String.format("Unable to log block debugging info: %s", e.getMessage()));
+		}
+	}
+
 }

src/main/java/org/qortal/block/BlockChain.java

@@ -71,6 +71,8 @@ public class BlockChain {
 
 	public enum FeatureTrigger {
 		atFindNextTransactionFix,
+		newBlockSigHeight,
+		shareBinFix,
 		calcChainWeightTimestamp;
 	}
 
@@ -377,6 +379,14 @@ public class BlockChain {
 		return this.featureTriggers.get(FeatureTrigger.atFindNextTransactionFix.name()).intValue();
 	}
 
+	public int getNewBlockSigHeight() {
+		return this.featureTriggers.get(FeatureTrigger.newBlockSigHeight.name()).intValue();
+	}
+
+	public int getShareBinFixHeight() {
+		return this.featureTriggers.get(FeatureTrigger.shareBinFix.name()).intValue();
+	}
+
 	public long getCalcChainWeightTimestamp() {
 		return this.featureTriggers.get(FeatureTrigger.calcChainWeightTimestamp.name()).longValue();
 	}

src/main/java/org/qortal/controller/BlockMinter.java

@@ -135,16 +135,19 @@ public class BlockMinter extends Thread {
 				// Disregard peers that have "misbehaved" recently
 				peers.removeIf(Controller.hasMisbehaved);
 
-				// Disregard peers that don't have a recent block
-				peers.removeIf(Controller.hasNoRecentBlock);
+				// Disregard peers that don't have a recent block, but only if we're not in recovery mode.
+				// In that mode, we want to allow minting on top of older blocks, to recover stalled networks.
+				if (Controller.getInstance().getRecoveryMode() == false)
+					peers.removeIf(Controller.hasNoRecentBlock);
 
 				// Don't mint if we don't have enough up-to-date peers as where would the transactions/consensus come from?
 				if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
 					continue;
 
-				// If our latest block isn't recent then we need to synchronize instead of minting.
+				// If our latest block isn't recent then we need to synchronize instead of minting, unless we're in recovery mode.
 				if (!peers.isEmpty() && lastBlockData.getTimestamp() < minLatestBlockTimestamp)
-					continue;
+					if (Controller.getInstance().getRecoveryMode() == false)
+						continue;
 
 				// There are enough peers with a recent block and our latest block is recent
 				// so go ahead and mint a block if possible.
@@ -165,6 +168,14 @@ public class BlockMinter extends Thread {
 				// Do we need to build any potential new blocks?
 				List<PrivateKeyAccount> newBlocksMintingAccounts = mintingAccountsData.stream().map(accountData -> new PrivateKeyAccount(repository, accountData.getPrivateKey())).collect(Collectors.toList());
 
+				// We might need to sit the next block out, if one of our minting accounts signed the previous one
+				final byte[] previousBlockMinter = previousBlockData.getMinterPublicKey();
+				final boolean mintedLastBlock = mintingAccountsData.stream().anyMatch(mintingAccount -> Arrays.equals(mintingAccount.getPublicKey(), previousBlockMinter));
+				if (mintedLastBlock) {
+					LOGGER.trace(String.format("One of our keys signed the last block, so we won't sign the next one"));
+					continue;
+				}
+
 				for (PrivateKeyAccount mintingAccount : newBlocksMintingAccounts) {
 					// First block does the AT heavy-lifting
 					if (newBlocks.isEmpty()) {
@@ -282,15 +293,17 @@ public class BlockMinter extends Thread {
 						RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(newBlock.getBlockData().getMinterPublicKey());
 
 						if (rewardShareData != null) {
-							LOGGER.info(String.format("Minted block %d, sig %.8s by %s on behalf of %s",
+							LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s on behalf of %s",
 									newBlock.getBlockData().getHeight(),
 									Base58.encode(newBlock.getBlockData().getSignature()),
+									Base58.encode(newBlock.getParent().getSignature()),
 									rewardShareData.getMinter(),
 									rewardShareData.getRecipient()));
 						} else {
-							LOGGER.info(String.format("Minted block %d, sig %.8s by %s",
+							LOGGER.info(String.format("Minted block %d, sig %.8s, parent sig: %.8s by %s",
 									newBlock.getBlockData().getHeight(),
 									Base58.encode(newBlock.getBlockData().getSignature()),
+									Base58.encode(newBlock.getParent().getSignature()),
 									newBlock.getMinter().getAddress()));
 						}
 

src/main/java/org/qortal/controller/Controller.java

@@ -67,8 +67,8 @@ import org.qortal.gui.SysTray;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.message.ArbitraryDataMessage;
-import org.qortal.network.message.BlockMessage;
 import org.qortal.network.message.BlockSummariesMessage;
+import org.qortal.network.message.CachedBlockMessage;
 import org.qortal.network.message.GetArbitraryDataMessage;
 import org.qortal.network.message.GetBlockMessage;
 import org.qortal.network.message.GetBlockSummariesMessage;
@@ -121,6 +121,7 @@ public class Controller extends Thread {
 	private static final long NTP_PRE_SYNC_CHECK_PERIOD = 5 * 1000L; // ms
 	private static final long NTP_POST_SYNC_CHECK_PERIOD = 5 * 60 * 1000L; // ms
 	private static final long DELETE_EXPIRED_INTERVAL = 5 * 60 * 1000L; // ms
+	private static final long RECOVERY_MODE_TIMEOUT = 10 * 60 * 1000L; // ms
 
 	// To do with online accounts list
 	private static final long ONLINE_ACCOUNTS_TASKS_INTERVAL = 10 * 1000L; // ms
@@ -143,16 +144,15 @@ public class Controller extends Thread {
 	private ExecutorService callbackExecutor = Executors.newFixedThreadPool(3);
 	private volatile boolean notifyGroupMembershipChange = false;
 
-	private static final int BLOCK_CACHE_SIZE = 10; // To cover typical Synchronizer request + a few spare
 	/** Latest blocks on our chain. Note: tail/last is the latest block. */
 	private final Deque<BlockData> latestBlocks = new LinkedList<>();
 
 	/** Cache of BlockMessages, indexed by block signature */
 	@SuppressWarnings("serial")
-	private final LinkedHashMap<ByteArray, BlockMessage> blockMessageCache = new LinkedHashMap<>() {
+	private final LinkedHashMap<ByteArray, CachedBlockMessage> blockMessageCache = new LinkedHashMap<>() {
 		@Override
-		protected boolean removeEldestEntry(Map.Entry<ByteArray, BlockMessage> eldest) {
-			return this.size() > BLOCK_CACHE_SIZE;
+		protected boolean removeEldestEntry(Map.Entry<ByteArray, CachedBlockMessage> eldest) {
+			return this.size() > Settings.getInstance().getBlockCacheSize();
 		}
 	};
 
@@ -176,6 +176,11 @@ public class Controller extends Thread {
 	/** Latest block signatures from other peers that we know are on inferior chains. */
 	List<ByteArray> inferiorChainSignatures = new ArrayList<>();
 
+	/** Recovery mode, which is used to bring back a stalled network */
+	private boolean recoveryMode = false;
+	private boolean peersAvailable = true; // peersAvailable must default to true
+	private long timePeersLastAvailable = 0;
+
 	/**
 	 * Map of recent requests for ARBITRARY transaction data payloads.
 	 * <p>
@@ -319,11 +324,12 @@ public class Controller extends Thread {
 		// Set initial chain height/tip
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			BlockData blockData = repository.getBlockRepository().getLastBlock();
+			int blockCacheSize = Settings.getInstance().getBlockCacheSize();
 
 			synchronized (this.latestBlocks) {
 				this.latestBlocks.clear();
 
-				for (int i = 0; i < BLOCK_CACHE_SIZE && blockData != null; ++i) {
+				for (int i = 0; i < blockCacheSize && blockData != null; ++i) {
 					this.latestBlocks.addFirst(blockData);
 					blockData = repository.getBlockRepository().fromHeight(blockData.getHeight() - 1);
 				}
@@ -358,6 +364,10 @@ public class Controller extends Thread {
 		}
 	}
 
+	public boolean getRecoveryMode() {
+		return this.recoveryMode;
+	}
+
 	// Entry point
 
 	public static void main(String[] args) {
@@ -536,12 +546,7 @@ public class Controller extends Thread {
 				if (now >= repositoryCheckpointTimestamp + repositoryCheckpointInterval) {
 					repositoryCheckpointTimestamp = now + repositoryCheckpointInterval;
 
-					if (Settings.getInstance().getShowCheckpointNotification())
-						SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_CHECKPOINT"),
-								Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_CHECKPOINT"),
-								MessageType.INFO);
-
-					RepositoryManager.checkpoint(true);
+					RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
 				}
 
 				// Give repository a chance to backup (if enabled)
@@ -634,6 +639,13 @@ public class Controller extends Thread {
 		// Disregard peers that don't have a recent block
 		peers.removeIf(hasNoRecentBlock);
 
+		checkRecoveryModeForPeers(peers);
+		if (recoveryMode) {
+			peers = Network.getInstance().getHandshakedPeers();
+			peers.removeIf(hasOnlyGenesisBlock);
+			peers.removeIf(hasMisbehaved);
+		}
+
 		// Check we have enough peers to potentially synchronize
 		if (peers.size() < Settings.getInstance().getMinBlockchainPeers())
 			return;
@@ -644,9 +656,31 @@ public class Controller extends Thread {
 		// Disregard peers that are on the same block as last sync attempt and we didn't like their chain
 		peers.removeIf(hasInferiorChainTip);
 
+		final int peersBeforeComparison = peers.size();
+
+		// Request recent block summaries from the remaining peers, and locate our common block with each
+		Synchronizer.getInstance().findCommonBlocksWithPeers(peers);
+
+		// Compare the peers against each other, and against our chain, which will return an updated list excluding those without common blocks
+		peers = Synchronizer.getInstance().comparePeers(peers);
+
+		// We may have added more inferior chain tips when comparing peers, so remove any peers that are currently on those chains
+		peers.removeIf(hasInferiorChainTip);
+
+		final int peersRemoved = peersBeforeComparison - peers.size();
+		if (peersRemoved > 0)
+			LOGGER.info(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
+
 		if (peers.isEmpty())
 			return;
 
+		if (peers.size() > 1) {
+			StringBuilder finalPeersString = new StringBuilder();
+			for (Peer peer : peers)
+				finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
+			LOGGER.info(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
+		}
+
 		// Pick random peer to sync with
 		int index = new SecureRandom().nextInt(peers.size());
 		Peer peer = peers.get(index);
@@ -749,6 +783,46 @@ public class Controller extends Thread {
 		}
 	}
 
+	private boolean checkRecoveryModeForPeers(List<Peer> qualifiedPeers) {
+		List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
+
+		if (handshakedPeers.size() > 0) {
+			// There is at least one handshaked peer
+			if (qualifiedPeers.isEmpty()) {
+				// There are no 'qualified' peers - i.e. peers that have a recent block we can sync to
+				boolean werePeersAvailable = peersAvailable;
+				peersAvailable = false;
+
+				// If peers only just became unavailable, update our record of the time they were last available
+				if (werePeersAvailable)
+					timePeersLastAvailable = NTP.getTime();
+
+				// If enough time has passed, enter recovery mode, which lifts some restrictions on who we can sync with and when we can mint
+				if (NTP.getTime() - timePeersLastAvailable > RECOVERY_MODE_TIMEOUT) {
+					if (recoveryMode == false) {
+						LOGGER.info(String.format("Peers have been unavailable for %d minutes. Entering recovery mode...", RECOVERY_MODE_TIMEOUT/60/1000));
+						recoveryMode = true;
+					}
+				}
+			} else {
+				// We now have at least one peer with a recent block, so we can exit recovery mode and sync normally
+				peersAvailable = true;
+				if (recoveryMode) {
+					LOGGER.info("Peers have become available again. Exiting recovery mode...");
+					recoveryMode = false;
+				}
+			}
+		}
+		return recoveryMode;
+	}
+
+	public void addInferiorChainSignature(byte[] inferiorSignature) {
+		// Update our list of inferior chain tips
+		ByteArray inferiorChainSignature = new ByteArray(inferiorSignature);
+		if (!inferiorChainSignatures.contains(inferiorChainSignature))
+			inferiorChainSignatures.add(inferiorChainSignature);
+	}
+
 	public static class StatusChangeEvent implements Event {
 		public StatusChangeEvent() {
 		}
@@ -780,7 +854,7 @@ public class Controller extends Thread {
 				actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED");
 		}
 
-		String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height);
+		String tooltip = String.format("%s - %d %s - %s %d", actionText, numberOfPeers, connectionsText, heightText, height) + "\n" + String.format("Build version: %s", this.buildVersion);
 		SysTray.getInstance().setToolTipText(tooltip);
 
 		this.callbackExecutor.execute(() -> {
@@ -811,7 +885,10 @@ public class Controller extends Thread {
 
 			repository.saveChanges();
 		} catch (DataException e) {
-			LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e);
+			if (RepositoryManager.isDeadlockRelated(e))
+				LOGGER.info("Couldn't delete some expired, unconfirmed transactions this round");
+			else
+				LOGGER.error("Repository issue while deleting expired unconfirmed transactions", e);
 		}
 	}
 
@@ -935,6 +1012,7 @@ public class Controller extends Thread {
 	public void onNewBlock(BlockData latestBlockData) {
 		// Protective copy
 		BlockData blockDataCopy = new BlockData(latestBlockData);
+		int blockCacheSize = Settings.getInstance().getBlockCacheSize();
 
 		synchronized (this.latestBlocks) {
 			BlockData cachedChainTip = this.latestBlocks.peekLast();
@@ -944,7 +1022,7 @@ public class Controller extends Thread {
 				this.latestBlocks.addLast(latestBlockData);
 
 				// Trim if necessary
-				if (this.latestBlocks.size() >= BLOCK_CACHE_SIZE)
+				if (this.latestBlocks.size() >= blockCacheSize)
 					this.latestBlocks.pollFirst();
 			} else {
 				if (cachedChainTip != null)
@@ -1152,14 +1230,15 @@ public class Controller extends Thread {
 
 		ByteArray signatureAsByteArray = new ByteArray(signature);
 
-		BlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray);
+		CachedBlockMessage cachedBlockMessage = this.blockMessageCache.get(signatureAsByteArray);
+		int blockCacheSize = Settings.getInstance().getBlockCacheSize();
 
 		// Check cached latest block message
 		if (cachedBlockMessage != null) {
 			this.stats.getBlockMessageStats.cacheHits.incrementAndGet();
 
 			// We need to duplicate it to prevent multiple threads setting ID on the same message
-			BlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId());
+			CachedBlockMessage clonedBlockMessage = cachedBlockMessage.cloneWithNewId(message.getId());
 
 			if (!peer.sendMessage(clonedBlockMessage))
 				peer.disconnect("failed to send block");
@@ -1187,15 +1266,18 @@ public class Controller extends Thread {
 
 			Block block = new Block(repository, blockData);
 
-			BlockMessage blockMessage = new BlockMessage(block);
+			CachedBlockMessage blockMessage = new CachedBlockMessage(block);
 			blockMessage.setId(message.getId());
 
 			// This call also causes the other needed data to be pulled in from repository
-			if (!peer.sendMessage(blockMessage))
+			if (!peer.sendMessage(blockMessage)) {
 				peer.disconnect("failed to send block");
+				// Don't fall-through to caching because failure to send might be from failure to build message
+				return;
+			}
 
 			// If request is for a recent block, cache it
-			if (getChainHeight() - blockData.getHeight() <= BLOCK_CACHE_SIZE) {
+			if (getChainHeight() - blockData.getHeight() <= blockCacheSize) {
 				this.stats.getBlockMessageStats.cacheFills.incrementAndGet();
 
 				this.blockMessageCache.put(new ByteArray(blockData.getSignature()), blockMessage);
@@ -1209,6 +1291,18 @@ public class Controller extends Thread {
 		TransactionMessage transactionMessage = (TransactionMessage) message;
 		TransactionData transactionData = transactionMessage.getTransactionData();
 
+		/*
+		 *  If we can't obtain blockchain lock immediately,
+		 *  e.g. Synchronizer is active, or another transaction is taking a while to validate,
+		 *  then we're using up a network thread for ages and clogging things up
+		 *  so bail out early
+		 */
+		ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+		if (!blockchainLock.tryLock()) {
+			LOGGER.trace(() -> String.format("Too busy to import %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
+			return;
+		}
+
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			Transaction transaction = Transaction.fromData(repository, transactionData);
 
@@ -1238,6 +1332,8 @@ public class Controller extends Thread {
 			LOGGER.debug(() -> String.format("Imported %s transaction %s from peer %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature()), peer));
 		} catch (DataException e) {
 			LOGGER.error(String.format("Repository issue while processing transaction %s from peer %s", Base58.encode(transactionData.getSignature()), peer), e);
+		} finally {
+			blockchainLock.unlock();
 		}
 	}
 

src/main/java/org/qortal/controller/Synchronizer.java

@@ -8,6 +8,7 @@ import java.util.Arrays;
 import java.util.List;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Collectors;
+import java.util.Iterator;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -17,6 +18,7 @@ import org.qortal.block.Block;
 import org.qortal.block.Block.ValidationResult;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
+import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -39,10 +41,23 @@ public class Synchronizer {
 
 	private static final Logger LOGGER = LogManager.getLogger(Synchronizer.class);
 
+	/** Max number of new blocks we aim to add to chain tip in each sync round */
+	private static final int SYNC_BATCH_SIZE = 200; // XXX move to Settings?
+
+	/** Initial jump back of block height when searching for common block with peer */
 	private static final int INITIAL_BLOCK_STEP = 8;
-	private static final int MAXIMUM_BLOCK_STEP = 500;
+	/** Maximum jump back of block height when searching for common block with peer */
+	private static final int MAXIMUM_BLOCK_STEP = 128;
+
+	/** Maximum difference in block height between tip and peer's common block before peer is considered TOO DIVERGENT */
 	private static final int MAXIMUM_COMMON_DELTA = 240; // XXX move to Settings?
-	private static final int SYNC_BATCH_SIZE = 200;
+
+	/** Maximum number of block signatures we ask from peer in one go */
+	private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
+
+	/** Number of retry attempts if a peer fails to respond with the requested data */
+	private static final int MAXIMUM_RETRIES = 2; // XXX move to Settings?
+
 
 	private static Synchronizer instance;
 
@@ -62,6 +77,377 @@ public class Synchronizer {
 		return instance;
 	}
 
+
+	/**
+	 * Iterate through a list of supplied peers, and attempt to find our common block with each.
+	 * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
+	 * <p>
+	 * Will return <tt>SynchronizationResult.OK</tt> on success.
+	 * <p>
+	 * @param peers
+	 * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong.
+	 * @throws InterruptedException
+	 */
+	public SynchronizationResult findCommonBlocksWithPeers(List<Peer> peers) throws InterruptedException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			try {
+
+				if (peers.size() == 0)
+					return SynchronizationResult.NOTHING_TO_DO;
+
+				// If our latest block is very old, it's best that we don't try and determine the best peers to sync to.
+				// This is because it can involve very large chain comparisons, which is too intensive.
+				// In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations.
+				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+				if (minLatestBlockTimestamp == null)
+					return SynchronizationResult.REPOSITORY_ISSUE;
+
+				final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+				if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
+					LOGGER.debug(String.format("Our latest block is very old, so we won't collect common block info from peers"));
+					return SynchronizationResult.NOTHING_TO_DO;
+				}
+
+				LOGGER.debug(String.format("Searching for common blocks with %d peers...", peers.size()));
+				final long startTime = System.currentTimeMillis();
+				int commonBlocksFound = 0;
+
+				for (Peer peer : peers) {
+					// Are we shutting down?
+					if (Controller.isStopping())
+						return SynchronizationResult.SHUTTING_DOWN;
+
+					// Check if we can use the cached common block data, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block
+					if (peer.canUseCachedCommonBlockData()) {
+						LOGGER.debug(String.format("Skipping peer %s because we already have the latest common block data in our cache. Cached common block sig is %.08s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature())));
+						continue;
+					}
+
+					// Cached data is stale, so clear it and repopulate
+					peer.setCommonBlockData(null);
+
+					// Search for the common block
+					Synchronizer.getInstance().findCommonBlockWithPeer(peer, repository);
+					if (peer.getCommonBlockData() != null)
+						commonBlocksFound++;
+				}
+
+				final long totalTimeTaken = System.currentTimeMillis() - startTime;
+				LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken));
+
+				return SynchronizationResult.OK;
+			} finally {
+				repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong
+			}
+		} catch (DataException e) {
+			LOGGER.error("Repository issue during synchronization with peer", e);
+			return SynchronizationResult.REPOSITORY_ISSUE;
+		}
+	}
+
+	/**
+	 * Attempt to find the find our common block with supplied peer.
+	 * If a common block is found, its summary will be retained in the peer's commonBlockSummary property, for processing later.
+	 * <p>
+	 * Will return <tt>SynchronizationResult.OK</tt> on success.
+	 * <p>
+	 * @param peer
+	 * @param repository
+	 * @return SynchronizationResult.OK if the process completed successfully, or a different SynchronizationResult if something went wrong.
+	 * @throws InterruptedException
+	 */
+	public SynchronizationResult findCommonBlockWithPeer(Peer peer, Repository repository) throws InterruptedException {
+		try {
+			final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+			final int ourInitialHeight = ourLatestBlockData.getHeight();
+
+			PeerChainTipData peerChainTipData = peer.getChainTipData();
+			int peerHeight = peerChainTipData.getLastHeight();
+			byte[] peersLastBlockSignature = peerChainTipData.getLastBlockSignature();
+
+			byte[] ourLastBlockSignature = ourLatestBlockData.getSignature();
+			LOGGER.debug(String.format("Fetching summaries from peer %s at height %d, sig %.8s, ts %d; our height %d, sig %.8s, ts %d", peer,
+					peerHeight, Base58.encode(peersLastBlockSignature), peer.getChainTipData().getLastBlockTimestamp(),
+					ourInitialHeight, Base58.encode(ourLastBlockSignature), ourLatestBlockData.getTimestamp()));
+
+			List<BlockSummaryData> peerBlockSummaries = new ArrayList<>();
+			SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, false, peerBlockSummaries);
+			if (findCommonBlockResult != SynchronizationResult.OK) {
+				// Logging performed by fetchSummariesFromCommonBlock() above
+				peer.setCommonBlockData(null);
+				return findCommonBlockResult;
+			}
+
+			// First summary is common block
+			final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature());
+			final BlockSummaryData commonBlockSummary = new BlockSummaryData(commonBlockData);
+			final int commonBlockHeight = commonBlockData.getHeight();
+			final byte[] commonBlockSig = commonBlockData.getSignature();
+			final String commonBlockSig58 = Base58.encode(commonBlockSig);
+			LOGGER.debug(String.format("Common block with peer %s is at height %d, sig %.8s, ts %d", peer,
+					commonBlockHeight, commonBlockSig58, commonBlockData.getTimestamp()));
+			peerBlockSummaries.remove(0);
+
+			// Store the common block summary against the peer, and the current chain tip (for caching)
+			peer.setCommonBlockData(new CommonBlockData(commonBlockSummary, peerChainTipData));
+
+			return SynchronizationResult.OK;
+		} catch (DataException e) {
+			LOGGER.error("Repository issue during synchronization with peer", e);
+			return SynchronizationResult.REPOSITORY_ISSUE;
+		}
+	}
+
+
+	/**
+	 * Compare a list of peers to determine the best peer(s) to sync to next.
+	 * <p>
+	 * Will return a filtered list of peers on success, or an identical list of peers on failure.
+	 * This allows us to fall back to legacy behaviour (random selection from the entire list of peers), if we are unable to make the comparison.
+	 * <p>
+	 * @param peers
+	 * @return a list of peers, possibly filtered.
+	 * @throws InterruptedException
+	 */
+	public List<Peer> comparePeers(List<Peer> peers) throws InterruptedException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			try {
+
+				// If our latest block is very old, it's best that we don't try and determine the best peers to sync to.
+				// This is because it can involve very large chain comparisons, which is too intensive.
+				// In reality, most forking problems occur near the chain tips, so we will reserve this functionality for those situations.
+				final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+				if (minLatestBlockTimestamp == null)
+					return peers;
+
+				final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+				if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
+					LOGGER.debug(String.format("Our latest block is very old, so we won't filter the peers list"));
+					return peers;
+				}
+
+				// Retrieve a list of unique common blocks from this list of peers
+				List<BlockSummaryData> commonBlocks = this.uniqueCommonBlocks(peers);
+
+				// Order common blocks by height, in ascending order
+				// This is essential for the logic below to make the correct decisions when discarding chains - do not remove
+				commonBlocks.sort((b1, b2) -> Integer.valueOf(b1.getHeight()).compareTo(Integer.valueOf(b2.getHeight())));
+
+				// Get our latest height
+				final int ourHeight = ourLatestBlockData.getHeight();
+
+				// Create a placeholder to track of common blocks that we can discard due to being inferior chains
+				int dropPeersAfterCommonBlockHeight = 0;
+
+				// Remove peers with no common block data
+				Iterator iterator = peers.iterator();
+				while (iterator.hasNext()) {
+					Peer peer = (Peer) iterator.next();
+					if (peer.getCommonBlockData() == null) {
+						LOGGER.debug(String.format("Removed peer %s because it has no common block data", peer));
+						iterator.remove();
+					}
+				}
+
+				// Loop through each group of common blocks
+				for (BlockSummaryData commonBlockSummary : commonBlocks) {
+					List<Peer> peersSharingCommonBlock = peers.stream().filter(peer -> peer.getCommonBlockData().getCommonBlockSummary().equals(commonBlockSummary)).collect(Collectors.toList());
+
+					// Check if we need to discard this group of peers
+					if (dropPeersAfterCommonBlockHeight > 0) {
+						if (commonBlockSummary.getHeight() > dropPeersAfterCommonBlockHeight) {
+							// We have already determined that the correct chain diverged from a lower height. We are safe to skip these peers.
+							for (Peer peer : peersSharingCommonBlock) {
+								LOGGER.debug(String.format("Peer %s has common block at height %d but the superior chain is at height %d. Removing it from this round.", peer, commonBlockSummary.getHeight(), dropPeersAfterCommonBlockHeight));
+								Controller.getInstance().addInferiorChainSignature(peer.getChainTipData().getLastBlockSignature());
+							}
+							continue;
+						}
+					}
+
+					// Calculate the length of the shortest peer chain sharing this common block, including our chain
+					final int ourAdditionalBlocksAfterCommonBlock = ourHeight - commonBlockSummary.getHeight();
+					int minChainLength = this.calculateMinChainLengthOfPeers(peersSharingCommonBlock, commonBlockSummary);
+
+					// Fetch block summaries from each peer
+					for (Peer peer : peersSharingCommonBlock) {
+
+						// If we're shutting down, just return the latest peer list
+						if (Controller.isStopping())
+							return peers;
+
+						// Count the number of blocks this peer has beyond our common block
+						final int peerHeight = peer.getChainTipData().getLastHeight();
+						final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
+						// Limit the number of blocks we are comparing. FUTURE: we could request more in batches, but there may not be a case when this is needed
+						int summariesRequired = Math.min(peerAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE);
+
+						// Check if we can use the cached common block summaries, by comparing the peer's current chain tip against the peer's chain tip when we last found our common block
+						boolean useCachedSummaries = false;
+						if (peer.canUseCachedCommonBlockData()) {
+							if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock() != null) {
+								if (peer.getCommonBlockData().getBlockSummariesAfterCommonBlock().size() == summariesRequired) {
+									LOGGER.debug(String.format("Using cached block summaries for peer %s", peer));
+									useCachedSummaries = true;
+								}
+							}
+						}
+
+						if (useCachedSummaries == false) {
+							if (summariesRequired > 0) {
+								LOGGER.trace(String.format("Requesting %d block summar%s from peer %s after common block %.8s. Peer height: %d", summariesRequired, (summariesRequired != 1 ? "ies" : "y"), peer, Base58.encode(commonBlockSummary.getSignature()), peerHeight));
+
+								List<BlockSummaryData> blockSummaries = this.getBlockSummaries(peer, commonBlockSummary.getSignature(), summariesRequired);
+								peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(blockSummaries);
+
+								if (blockSummaries != null) {
+									LOGGER.trace(String.format("Peer %s returned %d block summar%s", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y")));
+
+									// We need to adjust minChainLength if peers fail to return all expected block summaries
+									if (blockSummaries.size() < summariesRequired) {
+										// This could mean that the peer has re-orged. But we still have the same common block, so it's safe to proceed with this set of signatures instead.
+										LOGGER.debug(String.format("Peer %s returned %d block summar%s instead of expected %d", peer, blockSummaries.size(), (blockSummaries.size() != 1 ? "ies" : "y"), summariesRequired));
+
+										// Reduce minChainLength if we have at least 1 block for this peer. If we don't have any blocks, this peer will be excluded from chain weight comparisons later in the process, so we shouldn't update minChainLength
+										if (blockSummaries.size() > 0)
+											if (blockSummaries.size() < minChainLength)
+												minChainLength = blockSummaries.size();
+									}
+								}
+							} else {
+								// There are no block summaries after this common block
+								peer.getCommonBlockData().setBlockSummariesAfterCommonBlock(null);
+							}
+						}
+					}
+
+					// Fetch our corresponding block summaries. Limit to MAXIMUM_REQUEST_SIZE, in order to make the comparison fairer, as peers have been limited too
+					final int ourSummariesRequired = Math.min(ourAdditionalBlocksAfterCommonBlock, MAXIMUM_REQUEST_SIZE);
+					LOGGER.trace(String.format("About to fetch our block summaries from %d to %d. Our height: %d", commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired, ourHeight));
+					List<BlockSummaryData> ourBlockSummaries = repository.getBlockRepository().getBlockSummaries(commonBlockSummary.getHeight() + 1, commonBlockSummary.getHeight() + ourSummariesRequired);
+					if (ourBlockSummaries.isEmpty()) {
+						LOGGER.debug(String.format("We don't have any block summaries so can't compare our chain against peers with this common block. We can still compare them against each other."));
+					}
+					else {
+						populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
+						// Reduce minChainLength if we have less summaries
+						if (ourBlockSummaries.size() < minChainLength)
+							minChainLength = ourBlockSummaries.size();
+					}
+
+					// Create array to hold peers for comparison
+					List<Peer> superiorPeersForComparison = new ArrayList<>();
+
+					// Calculate max height for chain weight comparisons
+					int maxHeightForChainWeightComparisons = commonBlockSummary.getHeight() + minChainLength;
+
+					// Calculate our chain weight
+					BigInteger ourChainWeight = BigInteger.valueOf(0);
+					if (ourBlockSummaries.size() > 0)
+						ourChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), ourBlockSummaries, maxHeightForChainWeightComparisons);
+
+					NumberFormat formatter = new DecimalFormat("0.###E0");
+					NumberFormat accurateFormatter = new DecimalFormat("0.################E0");
+					LOGGER.debug(String.format("Our chain weight based on %d blocks is %s", ourBlockSummaries.size(), formatter.format(ourChainWeight)));
+
+					LOGGER.debug(String.format("Listing peers with common block %.8s...", Base58.encode(commonBlockSummary.getSignature())));
+					for (Peer peer : peersSharingCommonBlock) {
+						final int peerHeight = peer.getChainTipData().getLastHeight();
+						final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
+						final CommonBlockData peerCommonBlockData = peer.getCommonBlockData();
+
+						if (peerCommonBlockData == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock() == null || peerCommonBlockData.getBlockSummariesAfterCommonBlock().isEmpty()) {
+							// No response - remove this peer for now
+							LOGGER.debug(String.format("Peer %s doesn't have any block summaries - removing it from this round", peer));
+							peers.remove(peer);
+							continue;
+						}
+
+						final List<BlockSummaryData> peerBlockSummariesAfterCommonBlock = peerCommonBlockData.getBlockSummariesAfterCommonBlock();
+						populateBlockSummariesMinterLevels(repository, peerBlockSummariesAfterCommonBlock);
+
+						// Calculate cumulative chain weight of this blockchain subset, from common block to highest mutual block held by all peers in this group.
+						LOGGER.debug(String.format("About to calculate chain weight based on %d blocks for peer %s with common block %.8s (peer has %d blocks after common block)", peerBlockSummariesAfterCommonBlock.size(), peer, Base58.encode(commonBlockSummary.getSignature()), peerAdditionalBlocksAfterCommonBlock));
+						BigInteger peerChainWeight = Block.calcChainWeight(commonBlockSummary.getHeight(), commonBlockSummary.getSignature(), peerBlockSummariesAfterCommonBlock, maxHeightForChainWeightComparisons);
+						peer.getCommonBlockData().setChainWeight(peerChainWeight);
+						LOGGER.debug(String.format("Chain weight of peer %s based on %d blocks (%d - %d) is %s", peer, peerBlockSummariesAfterCommonBlock.size(), peerBlockSummariesAfterCommonBlock.get(0).getHeight(), peerBlockSummariesAfterCommonBlock.get(peerBlockSummariesAfterCommonBlock.size()-1).getHeight(), formatter.format(peerChainWeight)));
+
+						// Compare against our chain - if our blockchain has greater weight then don't synchronize with peer (or any others in this group)
+						if (ourChainWeight.compareTo(peerChainWeight) > 0) {
+							// This peer is on an inferior chain - remove it
+							LOGGER.debug(String.format("Peer %s is on an inferior chain to us - removing it from this round", peer));
+							peers.remove(peer);
+						}
+						else {
+							// Our chain is inferior
+							LOGGER.debug(String.format("Peer %s is on a better chain to us. We will compare the other peers sharing this common block against each other, and drop all peers sharing higher common blocks.", peer));
+							dropPeersAfterCommonBlockHeight = commonBlockSummary.getHeight();
+							superiorPeersForComparison.add(peer);
+						}
+					}
+
+					// Now that we have selected the best peers, compare them against each other and remove any with lower weights
+					if (superiorPeersForComparison.size() > 0) {
+						BigInteger bestChainWeight = null;
+						for (Peer peer : superiorPeersForComparison) {
+							// Increase bestChainWeight if needed
+							if (bestChainWeight == null || peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) >= 0)
+								bestChainWeight = peer.getCommonBlockData().getChainWeight();
+						}
+						for (Peer peer : superiorPeersForComparison) {
+							// Check if we should discard an inferior peer
+							if (peer.getCommonBlockData().getChainWeight().compareTo(bestChainWeight) < 0) {
+								BigInteger difference = bestChainWeight.subtract(peer.getCommonBlockData().getChainWeight());
+								LOGGER.debug(String.format("Peer %s has a lower chain weight (difference: %s) than other peer(s) in this group - removing it from this round.", peer, accurateFormatter.format(difference)));
+								peers.remove(peer);
+							}
+						}
+					}
+				}
+
+				return peers;
+			} finally {
+				repository.discardChanges(); // Free repository locks, if any, also in case anything went wrong
+			}
+		} catch (DataException e) {
+			LOGGER.error("Repository issue during peer comparison", e);
+			return peers;
+		}
+	}
+
+	private List<BlockSummaryData> uniqueCommonBlocks(List<Peer> peers) {
+		List<BlockSummaryData> commonBlocks = new ArrayList<>();
+
+		for (Peer peer : peers) {
+			if (peer.getCommonBlockData() != null && peer.getCommonBlockData().getCommonBlockSummary() != null) {
+				LOGGER.debug(String.format("Peer %s has common block %.8s", peer, Base58.encode(peer.getCommonBlockData().getCommonBlockSummary().getSignature())));
+
+				BlockSummaryData commonBlockSummary = peer.getCommonBlockData().getCommonBlockSummary();
+				if (!commonBlocks.contains(commonBlockSummary))
+					commonBlocks.add(commonBlockSummary);
+			}
+			else {
+				LOGGER.debug(String.format("Peer %s has no common block data. Skipping...", peer));
+			}
+		}
+
+		return commonBlocks;
+	}
+
+	private int calculateMinChainLengthOfPeers(List<Peer> peersSharingCommonBlock, BlockSummaryData commonBlockSummary) {
+		// Calculate the length of the shortest peer chain sharing this common block, including our chain
+		int minChainLength = 0;
+		for (Peer peer : peersSharingCommonBlock) {
+			final int peerHeight = peer.getChainTipData().getLastHeight();
+			final int peerAdditionalBlocksAfterCommonBlock = peerHeight - commonBlockSummary.getHeight();
+
+			if (peerAdditionalBlocksAfterCommonBlock < minChainLength || minChainLength == 0)
+				minChainLength = peerAdditionalBlocksAfterCommonBlock;
+		}
+		return minChainLength;
+	}
+
+
 	/**
 	 * Attempt to synchronize blockchain with peer.
 	 * <p>
@@ -97,9 +483,12 @@ public class Synchronizer {
 
 					List<BlockSummaryData> peerBlockSummaries = new ArrayList<>();
 					SynchronizationResult findCommonBlockResult = fetchSummariesFromCommonBlock(repository, peer, ourInitialHeight, force, peerBlockSummaries);
-					if (findCommonBlockResult != SynchronizationResult.OK)
+					if (findCommonBlockResult != SynchronizationResult.OK) {
 						// Logging performed by fetchSummariesFromCommonBlock() above
+						// Clear our common block cache for this peer
+						peer.setCommonBlockData(null);
 						return findCommonBlockResult;
+					}
 
 					// First summary is common block
 					final BlockData commonBlockData = repository.getBlockRepository().fromSignature(peerBlockSummaries.get(0).getSignature());
@@ -244,9 +633,13 @@ public class Synchronizer {
 		// Currently we work forward from common block until we hit a block we don't have
 		// TODO: rewrite as modified binary search!
 		int i;
-		for (i = 1; i < blockSummariesFromCommon.size(); ++i)
+		for (i = 1; i < blockSummariesFromCommon.size(); ++i) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
 			if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature()))
 				break;
+		}
 
 		// Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive
 		blockSummariesFromCommon.subList(0, i - 1).clear();
@@ -295,6 +688,9 @@ public class Synchronizer {
 
 				// Check peer sent valid heights
 				for (int i = 0; i < moreBlockSummaries.size(); ++i) {
+					if (Controller.isStopping())
+						return SynchronizationResult.SHUTTING_DOWN;
+
 					++lastSummaryHeight;
 
 					BlockSummaryData blockSummary = moreBlockSummaries.get(i);
@@ -316,7 +712,7 @@ public class Synchronizer {
 			populateBlockSummariesMinterLevels(repository, ourBlockSummaries);
 			populateBlockSummariesMinterLevels(repository, peerBlockSummaries);
 
-			final int mutualHeight = commonBlockHeight - 1 + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size());
+			final int mutualHeight = commonBlockHeight + Math.min(ourBlockSummaries.size(), peerBlockSummaries.size());
 
 			// Calculate cumulative chain weights of both blockchain subsets, from common block to highest mutual block.
 			BigInteger ourChainWeight = Block.calcChainWeight(commonBlockHeight, commonBlockSig, ourBlockSummaries, mutualHeight);
@@ -341,52 +737,154 @@ public class Synchronizer {
 		final byte[] commonBlockSig = commonBlockData.getSignature();
 		String commonBlockSig58 = Base58.encode(commonBlockSig);
 
+		byte[] latestPeerSignature = commonBlockSig;
+		int height = commonBlockHeight;
+
 		LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58));
 
-		int ourHeight = ourInitialHeight;
 
 		// Overall plan: fetch peer's blocks first, then orphan, then apply
 
 		// Convert any leftover (post-common) block summaries into signatures to request from peer
 		List<byte[]> peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList());
 
-		// Fetch remaining block signatures, if needed
-		int numberSignaturesRequired = peerBlockSignatures.size() - (peerHeight - commonBlockHeight);
-		if (numberSignaturesRequired > 0) {
-			byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1);
-
-			LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
-					numberSignaturesRequired, (numberSignaturesRequired != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature)));
-
-			List<byte[]> moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberSignaturesRequired);
-
-			if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) {
-				LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
-						ourHeight, Base58.encode(latestPeerSignature)));
-				return SynchronizationResult.NO_REPLY;
-			}
-
-			LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
-
-			peerBlockSignatures.addAll(moreBlockSignatures);
-		}
-
-		// Fetch blocks using signatures
-		LOGGER.debug(String.format("Fetching new blocks from peer %s", peer));
+		// Keep a list of blocks received so far
 		List<Block> peerBlocks = new ArrayList<>();
 
-		for (byte[] blockSignature : peerBlockSignatures) {
-			Block newBlock = this.fetchBlock(repository, peer, blockSignature);
+		// Calculate the total number of additional blocks this peer has beyond the common block
+		int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight;
+		// Subtract the number of signatures that we already have, as we don't need to request them again
+		int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size();
+
+		int retryCount = 0;
+		while (height < peerHeight) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
+            // Ensure we don't request more than MAXIMUM_REQUEST_SIZE
+            int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
+
+            // Do we need more signatures?
+			if (peerBlockSignatures.isEmpty() && numberRequested > 0) {
+                LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
+                        numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature)));
+
+                peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested);
+
+                if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) {
+                    LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
+                            height, Base58.encode(latestPeerSignature)));
+
+					// Clear our cache of common block summaries for this peer, as they are likely to be invalid
+					CommonBlockData cachedCommonBlockData = peer.getCommonBlockData();
+					if (cachedCommonBlockData != null)
+						cachedCommonBlockData.setBlockSummariesAfterCommonBlock(null);
+
+                    // If we have already received recent or newer blocks from this peer, go ahead and apply them
+                    if (peerBlocks.size() > 0) {
+						final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+                    	final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
+						final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+						if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) {
+
+							// If we have received at least one recent block, we can apply them
+							if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) {
+								LOGGER.debug("Newly received blocks are recent, so we will apply them");
+								break;
+							}
+
+							// If our latest block is very old....
+							if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
+								// ... and we have received a block that is more recent than our latest block ...
+								if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) {
+									// ... then apply the blocks, as it takes us a step forward.
+									// This is particularly useful when starting up a node that was on a small fork when it was last shut down.
+									// In these cases, we now allow the node to sync forward, and get onto the main chain again.
+									// Without this, we would require that the node syncs ENTIRELY with this peer,
+									// and any problems downloading a block would cause all progress to be lost.
+									LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp()));
+									break;
+								}
+							}
+						}
+                    }
+                    // Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state
+                    return SynchronizationResult.NO_REPLY;
+                }
+
+                numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size();
+                LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
+			}
+
+			if (peerBlockSignatures.isEmpty()) {
+				LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer));
+				break;
+			}
+
+			byte[] nextPeerSignature = peerBlockSignatures.get(0);
+			int nextHeight = height + 1;
+
+			LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer));
+			Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature);
 
 			if (newBlock == null) {
 				LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer,
-						ourHeight, Base58.encode(blockSignature)));
-				return SynchronizationResult.NO_REPLY;
+						nextHeight, Base58.encode(nextPeerSignature)));
+
+				if (retryCount >= MAXIMUM_RETRIES) {
+
+					// If we have already received recent or newer blocks from this peer, go ahead and apply them
+					if (peerBlocks.size() > 0) {
+						final BlockData ourLatestBlockData = repository.getBlockRepository().getLastBlock();
+						final Block peerLatestBlock = peerBlocks.get(peerBlocks.size() - 1);
+						final Long minLatestBlockTimestamp = Controller.getMinimumLatestBlockTimestamp();
+						if (ourLatestBlockData != null && peerLatestBlock != null && minLatestBlockTimestamp != null) {
+
+							// If we have received at least one recent block, we can apply them
+							if (peerLatestBlock.getBlockData().getTimestamp() > minLatestBlockTimestamp) {
+								LOGGER.debug("Newly received blocks are recent, so we will apply them");
+								break;
+							}
+
+							// If our latest block is very old....
+							if (ourLatestBlockData.getTimestamp() < minLatestBlockTimestamp) {
+								// ... and we have received a block that is more recent than our latest block ...
+								if (peerLatestBlock.getBlockData().getTimestamp() > ourLatestBlockData.getTimestamp()) {
+									// ... then apply the blocks, as it takes us a step forward.
+									// This is particularly useful when starting up a node that was on a small fork when it was last shut down.
+									// In these cases, we now allow the node to sync forward, and get onto the main chain again.
+									// Without this, we would require that the node syncs ENTIRELY with this peer,
+									// and any problems downloading a block would cause all progress to be lost.
+									LOGGER.debug(String.format("Newly received blocks are %d ms newer than our latest block - so we will apply them", peerLatestBlock.getBlockData().getTimestamp() - ourLatestBlockData.getTimestamp()));
+									break;
+								}
+							}
+						}
+					}
+					// Otherwise, give up and move on to the next peer, to avoid putting our chain into an outdated state
+					return SynchronizationResult.NO_REPLY;
+
+				} else {
+					// Re-fetch signatures, in case the peer is now on a different fork
+					peerBlockSignatures.clear();
+					numberSignaturesRequired = peerHeight - height;
+
+					// Retry until retryCount reaches MAXIMUM_RETRIES
+					retryCount++;
+					int triesRemaining = MAXIMUM_RETRIES - retryCount;
+					LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : "")));
+					continue;
+				}
 			}
 
+			// Reset retryCount because the last request succeeded
+			retryCount = 0;
+
+			LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer));
+
 			if (!newBlock.isSignatureValid()) {
 				LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer,
-						ourHeight, Base58.encode(blockSignature)));
+						nextHeight, Base58.encode(latestPeerSignature)));
 				return SynchronizationResult.INVALID_DATA;
 			}
 
@@ -395,12 +893,18 @@ public class Synchronizer {
 				transaction.setInitialApprovalStatus();
 
 			peerBlocks.add(newBlock);
+
+			// Now that we've received this block, we can increase our height and move on to the next one
+			latestPeerSignature = nextPeerSignature;
+			peerBlockSignatures.remove(0);
+			++height;
 		}
 
 		// Unwind to common block (unless common block is our latest block)
-		LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58));
+		int ourHeight = ourInitialHeight;
+		LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight));
 
-		BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight);
+		BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight);
 		while (ourHeight > commonBlockHeight) {
 			if (Controller.isStopping())
 				return SynchronizationResult.SHUTTING_DOWN;
@@ -422,10 +926,13 @@ public class Synchronizer {
 		LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer));
 
 		for (Block newBlock : peerBlocks) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
 			ValidationResult blockResult = newBlock.isValid();
 			if (blockResult != ValidationResult.OK) {
 				LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
-						ourHeight, Base58.encode(newBlock.getSignature()), blockResult.name()));
+						newBlock.getBlockData().getHeight(), Base58.encode(newBlock.getSignature()), blockResult.name()));
 				return SynchronizationResult.INVALID_DATA;
 			}
 
@@ -469,7 +976,8 @@ public class Synchronizer {
 
 			// Do we need more signatures?
 			if (peerBlockSignatures.isEmpty()) {
-				int numberRequested = maxBatchHeight - ourHeight;
+				int numberRequested = Math.min(maxBatchHeight - ourHeight, MAXIMUM_REQUEST_SIZE);
+
 				LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
 						numberRequested, (numberRequested != 1 ? "s": ""), ourHeight, Base58.encode(latestPeerSignature)));
 
@@ -488,7 +996,9 @@ public class Synchronizer {
 			peerBlockSignatures.remove(0);
 			++ourHeight;
 
+			LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer));
 			Block newBlock = this.fetchBlock(repository, peer, latestPeerSignature);
+			LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", ourHeight, Base58.encode(latestPeerSignature), peer));
 
 			if (newBlock == null) {
 				LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer,
@@ -571,6 +1081,9 @@ public class Synchronizer {
 		final int firstBlockHeight = blockSummaries.get(0).getHeight();
 
 		for (int i = 0; i < blockSummaries.size(); ++i) {
+			if (Controller.isStopping())
+				return;
+
 			BlockSummaryData blockSummary = blockSummaries.get(i);
 
 			// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level

src/main/java/org/qortal/controller/tradebot/AcctTradeBot.java

@@ -23,7 +23,7 @@ public interface AcctTradeBot {
 	public ResponseResult startResponse(Repository repository, ATData atData, ACCT acct,
 			CrossChainTradeData crossChainTradeData, String foreignKey, String receivingAddress) throws DataException;
 
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData);
+	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException;
 
 	public void progress(Repository repository, TradeBotData tradeBotData) throws DataException, ForeignBlockchainException;
 

src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java

@@ -345,11 +345,15 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 	}
 
 	@Override
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData) {
+	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException {
 		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
 		if (tradeBotState == null)
 			return true;
 
+		// If the AT doesn't exist then we might as well let the user tidy up
+		if (!repository.getATRepository().exists(tradeBotData.getAtAddress()))
+			return true;
+
 		switch (tradeBotState) {
 			case BOB_WAITING_FOR_AT_CONFIRM:
 			case ALICE_DONE:
@@ -378,7 +382,16 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 			// Attempt to fetch AT data
 			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
 			if (atData == null) {
-				LOGGER.warn(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
+				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
+
+				// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
+				// and so wipe the trade-bot entry
+				if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
+					LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
+					repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+					repository.saveChanges();
+				}
+
 				return;
 			}
 

src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java

@@ -211,6 +211,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 
 		TradeBot.updateTradeBotState(repository, tradeBotData, () -> String.format("Built AT %s. Waiting for deployment", atAddress));
 
+		// Attempt to backup the trade bot data
+		TradeBot.backupTradeBotData(repository);
+
 		// Return to user for signing and broadcast as we don't have their Qortal private key
 		try {
 			return DeployAtTransactionTransformer.toBytes(deployAtTransactionData);
@@ -283,6 +286,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 				tradeForeignPublicKey, tradeForeignPublicKeyHash,
 				crossChainTradeData.expectedForeignAmount, xprv58, null, lockTimeA, receivingPublicKeyHash);
 
+		// Attempt to backup the trade bot data
+		TradeBot.backupTradeBotData(repository);
+
 		// Check we have enough funds via xprv58 to fund P2SH to cover expectedForeignAmount
 		long p2shFee;
 		try {
@@ -343,11 +349,15 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 	}
 
 	@Override
-	public boolean canDelete(Repository repository, TradeBotData tradeBotData) {
+	public boolean canDelete(Repository repository, TradeBotData tradeBotData) throws DataException {
 		State tradeBotState = State.valueOf(tradeBotData.getStateValue());
 		if (tradeBotState == null)
 			return true;
 
+		// If the AT doesn't exist then we might as well let the user tidy up
+		if (!repository.getATRepository().exists(tradeBotData.getAtAddress()))
+			return true;
+
 		switch (tradeBotState) {
 			case BOB_WAITING_FOR_AT_CONFIRM:
 			case ALICE_DONE:
@@ -376,7 +386,16 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 			// Attempt to fetch AT data
 			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
 			if (atData == null) {
-				LOGGER.warn(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
+				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
+
+				// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
+				// and so wipe the trade-bot entry
+				if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
+					LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
+					repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
+					repository.saveChanges();
+				}
+
 				return;
 			}
 

src/main/java/org/qortal/controller/tradebot/TradeBot.java

@@ -7,6 +7,7 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Random;
+import java.util.concurrent.locks.ReentrantLock;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -267,6 +268,22 @@ public class TradeBot implements Listener {
 		return secret;
 	}
 
+	/*package*/ static void backupTradeBotData(Repository repository) {
+		// Attempt to backup the trade bot data. This an optional step and doesn't impact trading, so don't throw an exception on failure
+		try {
+			LOGGER.info("About to backup trade bot data...");
+			ReentrantLock blockchainLock = Controller.getInstance().getBlockchainLock();
+			blockchainLock.lockInterruptibly();
+			try {
+				repository.exportNodeLocalData(true);
+			} finally {
+				blockchainLock.unlock();
+			}
+		} catch (InterruptedException | DataException e) {
+			LOGGER.info(String.format("Failed to obtain blockchain lock when exporting trade bot data: %s", e.getMessage()));
+		}
+	}
+
 	/** Updates trade-bot entry to new state, with current timestamp, logs message and notifies state-change listeners. */
 	/*package*/ static void updateTradeBotState(Repository repository, TradeBotData tradeBotData,
 			String newState, int newStateValue, Supplier<String> logMessageSupplier) throws DataException {

src/main/java/org/qortal/crosschain/Bitcoin.java

@@ -42,35 +42,32 @@ public class Bitcoin extends Bitcoiny {
 			public Collection<ElectrumX.Server> getServers() {
 				return Arrays.asList(
 						// Servers chosen on NO BASIS WHATSOEVER from various sources!
-						new Server("enode.duckdns.org", Server.ConnectionType.SSL, 50002),
-						new Server("electrumx.ml", Server.ConnectionType.SSL, 50002),
-						new Server("electrum.bitkoins.nl", Server.ConnectionType.SSL, 50512),
-						new Server("btc.electroncash.dk", Server.ConnectionType.SSL, 60002),
-						new Server("electrumx.electricnewyear.net", Server.ConnectionType.SSL, 50002),
-						new Server("dxm.no-ip.biz", Server.ConnectionType.TCP, 50001),
-						new Server("kirsche.emzy.de", Server.ConnectionType.TCP, 50001),
-						new Server("2AZZARITA.hopto.org", Server.ConnectionType.TCP, 50001),
-						new Server("xtrum.com", Server.ConnectionType.TCP, 50001),
-						new Server("electrum.srvmin.network", Server.ConnectionType.TCP, 50001),
-						new Server("electrumx.alexridevski.net", Server.ConnectionType.TCP, 50001),
-						new Server("bitcoin.lukechilds.co", Server.ConnectionType.TCP, 50001),
-						new Server("electrum.poiuty.com", Server.ConnectionType.TCP, 50001),
-						new Server("horsey.cryptocowboys.net", Server.ConnectionType.TCP, 50001),
+						new Server("128.0.190.26", Server.ConnectionType.SSL, 50002),
+						new Server("hodlers.beer", Server.ConnectionType.SSL, 50002),
+						new Server("electrumx.erbium.eu", Server.ConnectionType.TCP, 50001),
+						new Server("electrumx.erbium.eu", Server.ConnectionType.SSL, 50002),
+						new Server("btc.lastingcoin.net", Server.ConnectionType.SSL, 50002),
+						new Server("electrum.bitaroo.net", Server.ConnectionType.SSL, 50002),
+						new Server("bitcoin.grey.pw", Server.ConnectionType.SSL, 50002),
+						new Server("2electrumx.hopto.me", Server.ConnectionType.SSL, 56022),
+						new Server("185.64.116.15", Server.ConnectionType.SSL, 50002),
+						new Server("kirsche.emzy.de", Server.ConnectionType.SSL, 50002),
+						new Server("alviss.coinjoined.com", Server.ConnectionType.SSL, 50002),
+						new Server("electrum.emzy.de", Server.ConnectionType.SSL, 50002),
 						new Server("electrum.emzy.de", Server.ConnectionType.TCP, 50001),
-						new Server("electrum-server.ninja", Server.ConnectionType.TCP, 50081),
-						new Server("bitcoin.electrumx.multicoin.co", Server.ConnectionType.TCP, 50001),
-						new Server("esx.geekhosters.com", Server.ConnectionType.TCP, 50001),
-						new Server("bitcoin.grey.pw", Server.ConnectionType.TCP, 50003),
-						new Server("exs.ignorelist.com", Server.ConnectionType.TCP, 50001),
-						new Server("electrum.coinext.com.br", Server.ConnectionType.TCP, 50001),
-						new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001),
-						new Server("skbxmit.coinjoined.com", Server.ConnectionType.TCP, 50001),
-						new Server("alviss.coinjoined.com", Server.ConnectionType.TCP, 50001),
-						new Server("electrum2.privateservers.network", Server.ConnectionType.TCP, 50001),
-						new Server("electrumx.schulzemic.net", Server.ConnectionType.TCP, 50001),
-						new Server("bitcoins.sk", Server.ConnectionType.TCP, 56001),
-						new Server("node.mendonca.xyz", Server.ConnectionType.TCP, 50001),
-						new Server("bitcoin.aranguren.org", Server.ConnectionType.TCP, 50001));
+						new Server("vmd71287.contaboserver.net", Server.ConnectionType.SSL, 50002),
+						new Server("btc.litepay.ch", Server.ConnectionType.SSL, 50002),
+						new Server("electrum.stippy.com", Server.ConnectionType.SSL, 50002),
+						new Server("xtrum.com", Server.ConnectionType.SSL, 50002),
+						new Server("electrum.acinq.co", Server.ConnectionType.SSL, 50002),
+						new Server("electrum2.taborsky.cz", Server.ConnectionType.SSL, 50002),
+						new Server("vmd63185.contaboserver.net", Server.ConnectionType.SSL, 50002),
+						new Server("electrum2.privateservers.network", Server.ConnectionType.SSL, 50002),
+						new Server("electrumx.alexridevski.net", Server.ConnectionType.SSL, 50002),
+						new Server("192.166.219.200", Server.ConnectionType.SSL, 50002),
+						new Server("2ex.digitaleveryware.com", Server.ConnectionType.SSL, 50002),
+						new Server("dxm.no-ip.biz", Server.ConnectionType.SSL, 50002),
+						new Server("caleb.vegas", Server.ConnectionType.SSL, 50002));
 			}
 
 			@Override
@@ -96,10 +93,8 @@ public class Bitcoin extends Bitcoiny {
 			@Override
 			public Collection<ElectrumX.Server> getServers() {
 				return Arrays.asList(
-						new Server("electrum.blockstream.info", Server.ConnectionType.TCP, 60001),
-						new Server("electrum.blockstream.info", Server.ConnectionType.SSL, 60002),
+						new Server("tn.not.fyi", Server.ConnectionType.SSL, 55002),
 						new Server("electrumx-test.1209k.com", Server.ConnectionType.SSL, 50002),
-						new Server("testnet.qtornado.com", Server.ConnectionType.TCP, 51001),
 						new Server("testnet.qtornado.com", Server.ConnectionType.SSL, 51002),
 						new Server("testnet.aranguren.org", Server.ConnectionType.TCP, 51001),
 						new Server("testnet.aranguren.org", Server.ConnectionType.SSL, 51002),

src/main/java/org/qortal/data/block/BlockSummaryData.java

@@ -2,6 +2,7 @@ package org.qortal.data.block;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Arrays;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 public class BlockSummaryData {
@@ -84,4 +85,21 @@ public class BlockSummaryData {
 		this.minterLevel = minterLevel;
 	}
 
+
+	@Override
+	public boolean equals(Object o) {
+		if (this == o)
+			return true;
+
+		if (o == null || getClass() != o.getClass())
+			return false;
+
+		BlockSummaryData otherBlockSummary = (BlockSummaryData) o;
+		if (this.getSignature() == null || otherBlockSummary.getSignature() == null)
+			return false;
+
+		// Treat two block summaries as equal if they have matching signatures
+		return Arrays.equals(this.getSignature(), otherBlockSummary.getSignature());
+	}
+
 }

src/main/java/org/qortal/data/block/CommonBlockData.java

@@ -0,0 +1,56 @@
+package org.qortal.data.block;
+
+import org.qortal.data.network.PeerChainTipData;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.math.BigInteger;
+import java.util.List;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class CommonBlockData {
+
+	// Properties
+	private BlockSummaryData commonBlockSummary = null;
+	private List<BlockSummaryData> blockSummariesAfterCommonBlock = null;
+	private BigInteger chainWeight = null;
+	private PeerChainTipData chainTipData = null;
+
+	// Constructors
+
+	protected CommonBlockData() {
+	}
+
+	public CommonBlockData(BlockSummaryData commonBlockSummary, PeerChainTipData chainTipData) {
+		this.commonBlockSummary = commonBlockSummary;
+		this.chainTipData = chainTipData;
+	}
+
+
+	// Getters / setters
+
+	public BlockSummaryData getCommonBlockSummary() {
+		return this.commonBlockSummary;
+	}
+
+	public List<BlockSummaryData> getBlockSummariesAfterCommonBlock() {
+		return this.blockSummariesAfterCommonBlock;
+	}
+
+	public void setBlockSummariesAfterCommonBlock(List<BlockSummaryData> blockSummariesAfterCommonBlock) {
+		this.blockSummariesAfterCommonBlock = blockSummariesAfterCommonBlock;
+	}
+
+	public BigInteger getChainWeight() {
+		return this.chainWeight;
+	}
+
+	public void setChainWeight(BigInteger chainWeight) {
+		this.chainWeight = chainWeight;
+	}
+
+	public PeerChainTipData getChainTipData() {
+		return this.chainTipData;
+	}
+
+}

src/main/java/org/qortal/network/Peer.java

@@ -15,6 +15,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Random;
+import java.util.Arrays;
 import java.util.concurrent.ArrayBlockingQueue;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.LinkedBlockingQueue;
@@ -22,6 +23,7 @@ import java.util.concurrent.TimeUnit;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.network.PeerData;
 import org.qortal.network.message.ChallengeMessage;
@@ -46,7 +48,7 @@ public class Peer {
 	private static final int CONNECT_TIMEOUT = 2000; // ms
 
 	/** Maximum time to wait for a message reply to arrive from peer. (ms) */
-	private static final int RESPONSE_TIMEOUT = 2000; // ms
+	private static final int RESPONSE_TIMEOUT = 3000; // ms
 
 	/**
 	 * Interval between PING messages to a peer. (ms)
@@ -106,6 +108,9 @@ public class Peer {
 	/** Latest block info as reported by peer. */
 	private PeerChainTipData peersChainTipData;
 
+	/** Our common block with this peer */
+	private CommonBlockData commonBlockData;
+
 	// Constructors
 
 	/** Construct unconnected, outbound Peer using socket address in peer data */
@@ -272,6 +277,18 @@ public class Peer {
 		}
 	}
 
+	public CommonBlockData getCommonBlockData() {
+		synchronized (this.peerInfoLock) {
+			return this.commonBlockData;
+		}
+	}
+
+	public void setCommonBlockData(CommonBlockData commonBlockData) {
+		synchronized (this.peerInfoLock) {
+			this.commonBlockData = commonBlockData;
+		}
+	}
+
 	/*package*/ void queueMessage(Message message) {
 		if (!this.pendingMessages.offer(message))
 			LOGGER.info(() -> String.format("No room to queue message from peer %s - discarding", this));
@@ -507,6 +524,7 @@ public class Peer {
 			}
 		} catch (MessageException e) {
 			LOGGER.warn(String.format("Failed to send %s message with ID %d to peer %s: %s", message.getType().name(), message.getId(), this, e.getMessage()));
+			return false;
 		} catch (IOException e) {
 			// Send failure
 			return false;
@@ -615,6 +633,25 @@ public class Peer {
 		}
 	}
 
+
+	// Common block data
+
+	public boolean canUseCachedCommonBlockData() {
+		PeerChainTipData peerChainTipData = this.getChainTipData();
+		CommonBlockData commonBlockData = this.getCommonBlockData();
+
+		if (peerChainTipData != null && commonBlockData != null) {
+			PeerChainTipData commonBlockChainTipData = commonBlockData.getChainTipData();
+			if (peerChainTipData.getLastBlockSignature() != null && commonBlockChainTipData != null && commonBlockChainTipData.getLastBlockSignature() != null) {
+				if (Arrays.equals(peerChainTipData.getLastBlockSignature(), commonBlockChainTipData.getLastBlockSignature())) {
+					return true;
+				}
+			}
+		}
+		return false;
+	}
+
+
 	// Utility methods
 
 	/** Returns true if ports and addresses (or hostnames) match */

src/main/java/org/qortal/network/message/CachedBlockMessage.java

@@ -0,0 +1,70 @@
+package org.qortal.network.message;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import org.qortal.block.Block;
+import org.qortal.transform.TransformationException;
+import org.qortal.transform.block.BlockTransformer;
+
+import com.google.common.primitives.Ints;
+
+// This is an OUTGOING-only Message which more readily lends itself to being cached
+public class CachedBlockMessage extends Message {
+
+	private Block block = null;
+	private byte[] cachedBytes = null;
+
+	public CachedBlockMessage(Block block) {
+		super(MessageType.BLOCK);
+
+		this.block = block;
+	}
+
+	private CachedBlockMessage(byte[] cachedBytes) {
+		super(MessageType.BLOCK);
+
+		this.block = null;
+		this.cachedBytes = cachedBytes;
+	}
+	
+	public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
+		throw new UnsupportedOperationException("CachedBlockMessage is for outgoing messages only");
+	}
+
+	@Override
+	protected byte[] toData() {
+		// Already serialized?
+		if (this.cachedBytes != null)
+			return cachedBytes;
+
+		if (this.block == null)
+			return null;
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(Ints.toByteArray(this.block.getBlockData().getHeight()));
+
+			bytes.write(BlockTransformer.toBytes(this.block));
+
+			this.cachedBytes = bytes.toByteArray();
+			// We no longer need source Block
+			// and Block contains repository handle which is highly likely to be invalid after this call
+			this.block = null;
+
+			return this.cachedBytes;
+		} catch (TransformationException | IOException e) {
+			return null;
+		}
+	}
+
+	public CachedBlockMessage cloneWithNewId(int newId) {
+		CachedBlockMessage clone = new CachedBlockMessage(this.cachedBytes);
+		clone.setId(newId);
+		return clone;
+	}
+
+}

src/main/java/org/qortal/repository/Repository.java

@@ -47,11 +47,9 @@ public interface Repository extends AutoCloseable {
 
 	public void backup(boolean quick) throws DataException;
 
-	public void checkpoint(boolean quick) throws DataException;
-
 	public void performPeriodicMaintenance() throws DataException;
 
-	public void exportNodeLocalData() throws DataException;
+	public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException;
 
 	public void importDataFromFile(String filename) throws DataException;
 

src/main/java/org/qortal/repository/RepositoryFactory.java

@@ -1,5 +1,7 @@
 package org.qortal.repository;
 
+import java.sql.SQLException;
+
 public interface RepositoryFactory {
 
 	public boolean wasPristineAtOpen();
@@ -12,4 +14,7 @@ public interface RepositoryFactory {
 
 	public void close() throws DataException;
 
+	// Not ideal place for this but implementating class will know the answer without having to open a new DB session
+	public boolean isDeadlockException(SQLException e);
+
 }

src/main/java/org/qortal/repository/RepositoryManager.java

@@ -1,9 +1,14 @@
 package org.qortal.repository;
 
+import java.sql.SQLException;
+
 public abstract class RepositoryManager {
 
 	private static RepositoryFactory repositoryFactory = null;
 
+	/** null if no checkpoint requested, TRUE for quick checkpoint, false for slow/full checkpoint. */
+	private static Boolean quickCheckpointRequested = null;
+
 	public static RepositoryFactory getRepositoryFactory() {
 		return repositoryFactory;
 	}
@@ -46,12 +51,12 @@ public abstract class RepositoryManager {
 		}
 	}
 
-	public static void checkpoint(boolean quick) {
-		try (final Repository repository = getRepository()) {
-			repository.checkpoint(quick);
-		} catch (DataException e) {
-			// Checkpoint is best-effort so don't complain
-		}
+	public static void setRequestedCheckpoint(Boolean quick) {
+		quickCheckpointRequested = quick;
+	}
+
+	public static Boolean getRequestedCheckpoint() {
+		return quickCheckpointRequested;
 	}
 
 	public static void rebuild() throws DataException {
@@ -66,4 +71,10 @@ public abstract class RepositoryManager {
 		repositoryFactory = oldRepositoryFactory.reopen();
 	}
 
+	public static boolean isDeadlockRelated(Throwable e) {
+		Throwable cause = e.getCause();
+
+		return SQLException.class.isInstance(cause) && repositoryFactory.isDeadlockException((SQLException) cause);
+	}
+
 }

src/main/java/org/qortal/repository/TransactionRepository.java

@@ -1,5 +1,6 @@
 package org.qortal.repository;
 
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
@@ -251,6 +252,14 @@ public interface TransactionRepository {
 	 */
 	public List<TransactionData> getUnconfirmedTransactions(TransactionType txType, byte[] creatorPublicKey) throws DataException;
 
+	/**
+	 * Returns list of unconfirmed transactions excluding specified type(s).
+	 * 
+	 * @return list of transactions, or empty if none.
+	 * @throws DataException
+	 */
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException;
+
 	/**
 	 * Remove transaction from unconfirmed transactions pile.
 	 * 

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepository.java

@@ -1,5 +1,6 @@
 package org.qortal.repository.hsqldb;
 
+import java.awt.TrayIcon.MessageType;
 import java.io.File;
 import java.io.IOException;
 import java.math.BigDecimal;
@@ -31,6 +32,8 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.crypto.Crypto;
+import org.qortal.globalization.Translator;
+import org.qortal.gui.SysTray;
 import org.qortal.repository.ATRepository;
 import org.qortal.repository.AccountRepository;
 import org.qortal.repository.ArbitraryRepository;
@@ -49,11 +52,17 @@ import org.qortal.repository.TransactionRepository;
 import org.qortal.repository.VotingRepository;
 import org.qortal.repository.hsqldb.transaction.HSQLDBTransactionRepository;
 import org.qortal.settings.Settings;
+import org.qortal.utils.NTP;
 
 public class HSQLDBRepository implements Repository {
 
 	private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepository.class);
 
+	private static final Object CHECKPOINT_LOCK = new Object();
+
+	// "serialization failure"
+	private static final Integer DEADLOCK_ERROR_CODE = Integer.valueOf(-4861);
+
 	protected Connection connection;
 	protected final Deque<Savepoint> savepoints = new ArrayDeque<>(3);
 	protected boolean debugState = false;
@@ -103,7 +112,10 @@ public class HSQLDBRepository implements Repository {
 			throw new DataException("Unable to fetch session ID from repository", e);
 		}
 
-		assertEmptyTransaction("connection creation");
+		// synchronize to block new connections if checkpointing in progress 
+		synchronized (CHECKPOINT_LOCK) {
+			assertEmptyTransaction("connection creation");
+		}
 	}
 
 	// Getters / setters
@@ -284,6 +296,9 @@ public class HSQLDBRepository implements Repository {
 			this.sqlStatements = null;
 			this.savepoints.clear();
 
+			// If a checkpoint has been requested, we could perform that now
+			this.maybeCheckpoint();
+
 			// Give connection back to the pool
 			this.connection.close();
 			this.connection = null;
@@ -292,6 +307,58 @@ public class HSQLDBRepository implements Repository {
 		}
 	}
 
+	private void maybeCheckpoint() throws DataException {
+		// To serialize checkpointing and to block new sessions when checkpointing in progress
+		synchronized (CHECKPOINT_LOCK) {
+			Boolean quickCheckpointRequest = RepositoryManager.getRequestedCheckpoint();
+			if (quickCheckpointRequest == null)
+				return;
+
+			// We can only perform a CHECKPOINT if no other HSQLDB session is mid-transaction,
+			// otherwise the CHECKPOINT blocks for COMMITs and other threads can't open HSQLDB sessions
+			// due to HSQLDB blocking until CHECKPOINT finishes - i.e. deadlock
+			String sql = "SELECT COUNT(*) "
+					+ "FROM Information_schema.system_sessions "
+					+ "WHERE transaction = TRUE";
+
+			try {
+				PreparedStatement pstmt = this.cachePreparedStatement(sql);
+
+				if (!pstmt.execute())
+					throw new DataException("Unable to check repository session status");
+
+				try (ResultSet resultSet = pstmt.getResultSet()) {
+					if (resultSet == null || !resultSet.next())
+						// Failed to even find HSQLDB session info!
+						throw new DataException("No results when checking repository session status");
+
+					int transactionCount = resultSet.getInt(1);
+
+					if (transactionCount > 0)
+						// We can't safely perform CHECKPOINT due to ongoing SQL transactions
+						return;
+				}
+
+				LOGGER.info("Performing repository CHECKPOINT...");
+
+				if (Settings.getInstance().getShowCheckpointNotification())
+					SysTray.getInstance().showMessage(Translator.INSTANCE.translate("SysTray", "DB_CHECKPOINT"),
+							Translator.INSTANCE.translate("SysTray", "PERFORMING_DB_CHECKPOINT"),
+							MessageType.INFO);
+
+				try (Statement stmt = this.connection.createStatement()) {
+					stmt.execute(Boolean.TRUE.equals(quickCheckpointRequest) ? "CHECKPOINT" : "CHECKPOINT DEFRAG");
+				}
+
+				// Completed!
+				LOGGER.info("Repository CHECKPOINT completed!");
+				RepositoryManager.setRequestedCheckpoint(null);
+			} catch (SQLException e) {
+				throw new DataException("Unable to check repository session status", e);
+			}
+		}
+	}
+
 	@Override
 	public void rebuild() throws DataException {
 		LOGGER.info("Rebuilding repository from scratch");
@@ -379,15 +446,6 @@ public class HSQLDBRepository implements Repository {
 		}
 	}
 
-	@Override
-	public void checkpoint(boolean quick) throws DataException {
-		try (Statement stmt = this.connection.createStatement()) {
-			stmt.execute(quick ? "CHECKPOINT" : "CHECKPOINT DEFRAG");
-		} catch (SQLException e) {
-			throw new DataException("Unable to perform repository checkpoint");
-		}
-	}
-
 	@Override
 	public void performPeriodicMaintenance() throws DataException {
 		// Defrag DB - takes a while!
@@ -402,10 +460,44 @@ public class HSQLDBRepository implements Repository {
 	}
 
 	@Override
-	public void exportNodeLocalData() throws DataException {
+	public void exportNodeLocalData(boolean keepArchivedCopy) throws DataException {
+
+		// Create the qortal-backup folder if it doesn't exist
+		Path backupPath = Paths.get("qortal-backup");
+		try {
+			Files.createDirectories(backupPath);
+		} catch (IOException e) {
+			LOGGER.info("Unable to create backup folder");
+			throw new DataException("Unable to create backup folder");
+		}
+
+		// We need to rename or delete an existing TradeBotStates backup before creating a new one
+		File tradeBotStatesBackupFile = new File("qortal-backup/TradeBotStates.script");
+		if (tradeBotStatesBackupFile.exists()) {
+			if (keepArchivedCopy) {
+				// Rename existing TradeBotStates backup, to make sure that we're not overwriting any keys
+				File archivedBackupFile = new File(String.format("qortal-backup/TradeBotStates-archive-%d.script", NTP.getTime()));
+				if (tradeBotStatesBackupFile.renameTo(archivedBackupFile))
+					LOGGER.info(String.format("Moved existing TradeBotStates backup file to %s", archivedBackupFile.getPath()));
+				else
+					throw new DataException("Unable to rename existing TradeBotStates backup");
+			} else {
+				// Delete existing copy
+				LOGGER.info("Deleting existing TradeBotStates backup because it is being replaced with a new one");
+				tradeBotStatesBackupFile.delete();
+			}
+		}
+
+		// There's currently no need to take an archived copy of the MintingAccounts data - just delete the old one if it exists
+		File mintingAccountsBackupFile = new File("qortal-backup/MintingAccounts.script");
+		if (mintingAccountsBackupFile.exists()) {
+			LOGGER.info("Deleting existing MintingAccounts backup because it is being replaced with a new one");
+			mintingAccountsBackupFile.delete();
+		}
+
 		try (Statement stmt = this.connection.createStatement()) {
-			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'MintingAccounts.script'");
-			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'TradeBotStates.script'");
+			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE MintingAccounts DATA TO 'qortal-backup/MintingAccounts.script'");
+			stmt.execute("PERFORM EXPORT SCRIPT FOR TABLE TradeBotStates DATA TO 'qortal-backup/TradeBotStates.script'");
 			LOGGER.info("Exported sensitive/node-local data: minting keys and trade bot states");
 		} catch (SQLException e) {
 			throw new DataException("Unable to export sensitive/node-local data from repository");
@@ -418,12 +510,12 @@ public class HSQLDBRepository implements Repository {
 			LOGGER.info(() -> String.format("Importing data into repository from %s", filename));
 
 			String escapedFilename = stmt.enquoteLiteral(filename);
-			stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " STOP ON ERROR");
+			stmt.execute("PERFORM IMPORT SCRIPT DATA FROM " + escapedFilename + " CONTINUE ON ERROR");
 
 			LOGGER.info(() -> String.format("Imported data into repository from %s", filename));
 		} catch (SQLException e) {
 			LOGGER.info(() -> String.format("Failed to import data into repository from %s: %s", filename, e.getMessage()));
-			throw new DataException("Unable to export sensitive/node-local data from repository: " + e.getMessage());
+			throw new DataException("Unable to import sensitive/node-local data to repository: " + e.getMessage());
 		}
 	}
 
@@ -624,7 +716,7 @@ public class HSQLDBRepository implements Repository {
 	/**
 	 * Execute PreparedStatement and return changed row count.
 	 * 
-	 * @param preparedStatement
+	 * @param sql
 	 * @param objects
 	 * @return number of changed rows
 	 * @throws SQLException
@@ -636,8 +728,8 @@ public class HSQLDBRepository implements Repository {
 	/**
 	 * Execute batched PreparedStatement
 	 * 
-	 * @param preparedStatement
-	 * @param objects
+	 * @param sql
+	 * @param batchedObjects
 	 * @return number of changed rows
 	 * @throws SQLException
 	 */
@@ -654,7 +746,16 @@ public class HSQLDBRepository implements Repository {
 
 		long beforeQuery = this.slowQueryThreshold == null ? 0 : System.currentTimeMillis();
 
-		int[] updateCounts = preparedStatement.executeBatch();
+		int[] updateCounts = null;
+		try {
+			updateCounts = preparedStatement.executeBatch();
+		} catch (SQLException e) {
+			if (isDeadlockException(e))
+				// We want more info on what other DB sessions are doing to cause this
+				examineException(e);
+
+			throw e;
+		}
 
 		if (this.slowQueryThreshold != null) {
 			long queryTime = System.currentTimeMillis() - beforeQuery;
@@ -752,7 +853,7 @@ public class HSQLDBRepository implements Repository {
 	 * 
 	 * @param tableName
 	 * @param whereClause
-	 * @param objects
+	 * @param batchedObjects
 	 * @throws SQLException
 	 */
 	public int deleteBatch(String tableName, String whereClause, List<Object[]> batchedObjects) throws SQLException {
@@ -865,6 +966,8 @@ public class HSQLDBRepository implements Repository {
 
 	/** Logs other HSQLDB sessions then returns passed exception */
 	public SQLException examineException(SQLException e) {
+		// TODO: could log at DEBUG for deadlocks by checking RepositoryManager.isDeadlockRelated(e)?
+
 		LOGGER.error(() -> String.format("[Session %d] HSQLDB error: %s", this.sessionId, e.getMessage()), e);
 
 		logStatements();
@@ -946,4 +1049,8 @@ public class HSQLDBRepository implements Repository {
 		return Crypto.toAddress(publicKey);
 	}
 
+	/*package*/ static boolean isDeadlockException(SQLException e) {
+		return DEADLOCK_ERROR_CODE.equals(e.getErrorCode());
+	}
+
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBRepositoryFactory.java

@@ -14,11 +14,11 @@ import org.hsqldb.jdbc.HSQLDBPool;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryFactory;
+import org.qortal.settings.Settings;
 
 public class HSQLDBRepositoryFactory implements RepositoryFactory {
 
 	private static final Logger LOGGER = LogManager.getLogger(HSQLDBRepositoryFactory.class);
-	private static final int POOL_SIZE = 100;
 
 	/** Log getConnection() calls that take longer than this. (ms) */
 	private static final long SLOW_CONNECTION_THRESHOLD = 1000L;
@@ -57,7 +57,7 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 			HSQLDBRepository.attemptRecovery(connectionUrl);
 		}
 
-		this.connectionPool = new HSQLDBPool(POOL_SIZE);
+		this.connectionPool = new HSQLDBPool(Settings.getInstance().getRepositoryConnectionPoolSize());
 		this.connectionPool.setUrl(this.connectionUrl);
 
 		Properties properties = new Properties();
@@ -94,7 +94,11 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 	@Override
 	public Repository tryRepository() throws DataException {
 		try {
-			return new HSQLDBRepository(this.tryConnection());
+			Connection connection = this.tryConnection();
+			if (connection == null)
+				return null;
+
+			return new HSQLDBRepository(connection);
 		} catch (SQLException e) {
 			throw new DataException("Repository instantiation error", e);
 		}
@@ -144,4 +148,9 @@ public class HSQLDBRepositoryFactory implements RepositoryFactory {
 		}
 	}
 
+	@Override
+	public boolean isDeadlockException(SQLException e) {
+		return HSQLDBRepository.isDeadlockException(e);
+	}
+
 }

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java

@@ -9,6 +9,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.EnumMap;
+import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 
@@ -1181,6 +1182,51 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		}
 	}
 
+	@Override
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException {
+		StringBuilder sql = new StringBuilder(1024);
+		sql.append("SELECT signature FROM UnconfirmedTransactions ");
+		sql.append("JOIN Transactions USING (signature) ");
+		sql.append("WHERE type NOT IN (");
+
+		boolean firstTxType = true;
+		for (TransactionType txType : excludedTxTypes) {
+			if (firstTxType)
+				firstTxType = false;
+			else
+				sql.append(", ");
+
+			sql.append(txType.value);
+		}
+
+		sql.append(")");
+		sql.append("ORDER BY created_when, signature");
+
+		List<TransactionData> transactions = new ArrayList<>();
+
+		// Find transactions with no corresponding row in BlockTransactions
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
+			if (resultSet == null)
+				return transactions;
+
+			do {
+				byte[] signature = resultSet.getBytes(1);
+
+				TransactionData transactionData = this.fromSignature(signature);
+
+				if (transactionData == null)
+					// Something inconsistent with the repository
+					throw new DataException(String.format("Unable to fetch unconfirmed transaction %s from repository?", Base58.encode(signature)));
+
+				transactions.add(transactionData);
+			} while (resultSet.next());
+
+			return transactions;
+		} catch (SQLException | DataException e) {
+			throw new DataException("Unable to fetch unconfirmed transactions from repository", e);
+		}
+	}
+
 	@Override
 	public void confirmTransaction(byte[] signature) throws DataException {
 		try {

src/main/java/org/qortal/settings/Settings.java

@@ -52,7 +52,7 @@ public class Settings {
 	// UI servers
 	private int uiPort = 12388;
 	private String[] uiLocalServers = new String[] {
-		"localhost", "127.0.0.1", "172.24.1.1", "qor.tal"
+		"localhost", "127.0.0.1"
 	};
 	private String[] uiRemoteServers = new String[] {
 		"node1.qortal.org", "node2.qortal.org", "node3.qortal.org", "node4.qortal.org", "node5.qortal.org",
@@ -89,6 +89,8 @@ public class Settings {
 	private long repositoryCheckpointInterval = 60 * 60 * 1000L; // 1 hour (ms) default
 	/** Whether to show a notification when we perform repository 'checkpoint'. */
 	private boolean showCheckpointNotification = false;
+	/* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */
+	private int blockCacheSize = 10;
 
 	/** How long to keep old, full, AT state data (ms). */
 	private long atStatesMaxLifetime = 2 * 7 * 24 * 60 * 60 * 1000L; // milliseconds
@@ -134,6 +136,8 @@ public class Settings {
 	private Long slowQueryThreshold = null;
 	/** Repository storage path. */
 	private String repositoryPath = "db";
+	/** Repository connection pool size. Needs to be a bit bigger than maxNetworkThreadPoolSize */
+	private int repositoryConnectionPoolSize = 100;
 
 	// Auto-update sources
 	private String[] autoUpdateRepos = new String[] {
@@ -361,6 +365,10 @@ public class Settings {
 		return this.maxTransactionTimestampFuture;
 	}
 
+	public int getBlockCacheSize() {
+		return this.blockCacheSize;
+	}
+
 	public boolean isTestNet() {
 		return this.isTestNet;
 	}
@@ -424,6 +432,10 @@ public class Settings {
 		return this.repositoryPath;
 	}
 
+	public int getRepositoryConnectionPoolSize() {
+		return this.repositoryConnectionPoolSize;
+	}
+
 	public boolean isAutoUpdateEnabled() {
 		return this.autoUpdateEnabled;
 	}

src/main/java/org/qortal/transaction/Transaction.java

@@ -4,6 +4,7 @@ import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Comparator;
+import java.util.EnumSet;
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
@@ -605,7 +606,8 @@ public abstract class Transaction {
 	public static List<TransactionData> getUnconfirmedTransactions(Repository repository) throws DataException {
 		BlockData latestBlockData = repository.getBlockRepository().getLastBlock();
 
-		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions();
+		EnumSet<TransactionType> excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE);
+		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes);
 
 		unconfirmedTransactions.sort(getDataComparator());
 

src/main/java/org/qortal/transform/block/BlockTransformer.java

@@ -326,24 +326,36 @@ public class BlockTransformer extends Transformer {
 		}
 	}
 
-	public static byte[] getMinterSignatureFromReference(byte[] blockReference) {
-		return Arrays.copyOf(blockReference, MINTER_SIGNATURE_LENGTH);
+	private static byte[] getReferenceBytesForMinterSignature(int blockHeight, byte[] reference) {
+		int newBlockSigTriggerHeight = BlockChain.getInstance().getNewBlockSigHeight();
+
+		return blockHeight >= newBlockSigTriggerHeight
+				// 'new' block sig uses all of previous block's signature
+				? reference
+				// 'old' block sig only uses first 64 bytes of previous block's signature
+				: Arrays.copyOf(reference, MINTER_SIGNATURE_LENGTH);
 	}
 
-	public static byte[] getBytesForMinterSignature(BlockData blockData) throws TransformationException {
-		byte[] minterSignature = getMinterSignatureFromReference(blockData.getReference());
+	public static byte[] getBytesForMinterSignature(BlockData blockData) {
+		byte[] referenceBytes = getReferenceBytesForMinterSignature(blockData.getHeight(), blockData.getReference());
 
-		return getBytesForMinterSignature(minterSignature, blockData.getMinterPublicKey(), blockData.getEncodedOnlineAccounts());
+		return getBytesForMinterSignature(referenceBytes, blockData.getMinterPublicKey(), blockData.getEncodedOnlineAccounts());
 	}
 
-	public static byte[] getBytesForMinterSignature(byte[] minterSignature, byte[] minterPublicKey, byte[] encodedOnlineAccounts) {
-		byte[] bytes = new byte[MINTER_SIGNATURE_LENGTH + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length];
+	public static byte[] getBytesForMinterSignature(BlockData parentBlockData, byte[] minterPublicKey, byte[] encodedOnlineAccounts) {
+		byte[] referenceBytes = getReferenceBytesForMinterSignature(parentBlockData.getHeight() + 1, parentBlockData.getSignature());
 
-		System.arraycopy(minterSignature, 0, bytes, 0, MINTER_SIGNATURE_LENGTH);
+		return getBytesForMinterSignature(referenceBytes, minterPublicKey, encodedOnlineAccounts);
+	}
 
-		System.arraycopy(minterPublicKey, 0, bytes, MINTER_SIGNATURE_LENGTH, MINTER_PUBLIC_KEY_LENGTH);
+	private static byte[] getBytesForMinterSignature(byte[] referenceBytes, byte[] minterPublicKey, byte[] encodedOnlineAccounts) {
+		byte[] bytes = new byte[referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH + encodedOnlineAccounts.length];
 
-		System.arraycopy(encodedOnlineAccounts, 0, bytes, MINTER_SIGNATURE_LENGTH + MINTER_PUBLIC_KEY_LENGTH, encodedOnlineAccounts.length);
+		System.arraycopy(referenceBytes, 0, bytes, 0, referenceBytes.length);
+
+		System.arraycopy(minterPublicKey, 0, bytes, referenceBytes.length, MINTER_PUBLIC_KEY_LENGTH);
+
+		System.arraycopy(encodedOnlineAccounts, 0, bytes, referenceBytes.length + MINTER_PUBLIC_KEY_LENGTH, encodedOnlineAccounts.length);
 
 		return bytes;
 	}