Bump version to 1.4.6

Main differences / improvements:
- Only request a single batch of signatures upfront, instead of the entire peer's chain. There is no point in requesting them all, as the later ones may not be valid by the time we have finished requesting all the blocks before them.
- If we fail to fetch a block, clear any queued signatures that are in memory and re-fetch signatures after the last block received. This allows us to cope with peers that re-org whilst we are syncing with them.
- If we can't find any more block signatures, or the peer fails to respond to a block, apply our progress anyway. This should reduce wasted work and network congestion, and helps cope with larger peer re-orgs.
- The retry mechanism remains in place, but instead of fetching the same incorrect block over and over, it will attempt to locate a new block signature each time, as described above. To help reduce code complexity, block signature requests are no longer retried.

WindowsInstaller/Qortal.aip

@@ -19,10 +19,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{9EA6B58D-641E-442E-8F16-5D35B92B9F9B} 1049:{B16722F6-C2FA-418D-A9DA-69707FE2034B} 2052:{459AC873-98DC-43AC-8787-B23BAF976FF5} 2057:{CD923B63-65A0-4F2A-93B6-6362AAC8608E} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{DE0C17EF-F9D2-48A3-A0D7-FF7BB01698A2} 1049:{89129011-9CE7-44AD-97EC-152371A87D58} 2052:{C93D5260-C6BD-4341-A96D-7D878EE6D59B} 2057:{AE6EDB44-3120-4588-8C9A-558F1BE42D73} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="1.4.2" Type="32"/>
+    <ROW Property="ProductVersion" Value="1.4.5" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -174,7 +174,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_97" ComponentId="{D5544706-E2A7-424F-AEA5-3963E355AA29}" Directory_="jdk.crypto.mscapi_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_97" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_98" ComponentId="{104DBCE8-A458-4B3E-9EFA-2D8613561619}" Directory_="jdk.dynalink_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_98" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_99" ComponentId="{D02E3C37-E81A-48FA-9E28-B26B728AECD9}" Directory_="jdk.httpserver_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_99" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{F9375D31-26C0-4E23-948D-3570B43B7FA2}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{CFB6FB43-13A6-4A4F-BCF9-7698463CE032}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="DATA_PATH" ComponentId="{EE0B6107-E244-4CDB-B195-E9038D2F1E0E}" Directory_="DATA_PATH" Attributes="0"/>

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>1.4.5</version>
+	<version>1.4.6</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/controller/Synchronizer.java

@@ -54,7 +54,7 @@ public class Synchronizer {
 	private static final int MAXIMUM_REQUEST_SIZE = 200; // XXX move to Settings?
 
 	/** Number of retry attempts if a peer fails to respond with the requested data */
-	private static final int MAXIMUM_RETRIES = 3; // XXX move to Settings?
+	private static final int MAXIMUM_RETRIES = 1; // XXX move to Settings?
 
 
 	private static Synchronizer instance;
@@ -257,9 +257,13 @@ public class Synchronizer {
 		// Currently we work forward from common block until we hit a block we don't have
 		// TODO: rewrite as modified binary search!
 		int i;
-		for (i = 1; i < blockSummariesFromCommon.size(); ++i)
+		for (i = 1; i < blockSummariesFromCommon.size(); ++i) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
 			if (!repository.getBlockRepository().exists(blockSummariesFromCommon.get(i).getSignature()))
 				break;
+		}
 
 		// Note: index i - 1 isn't cleared: List.subList is fromIndex inclusive to toIndex exclusive
 		blockSummariesFromCommon.subList(0, i - 1).clear();
@@ -308,6 +312,9 @@ public class Synchronizer {
 
 				// Check peer sent valid heights
 				for (int i = 0; i < moreBlockSummaries.size(); ++i) {
+					if (Controller.isStopping())
+						return SynchronizationResult.SHUTTING_DOWN;
+
 					++lastSummaryHeight;
 
 					BlockSummaryData blockSummary = moreBlockSummaries.get(i);
@@ -354,45 +361,89 @@ public class Synchronizer {
 		final byte[] commonBlockSig = commonBlockData.getSignature();
 		String commonBlockSig58 = Base58.encode(commonBlockSig);
 
+		byte[] latestPeerSignature = commonBlockSig;
+		int height = commonBlockHeight;
+
 		LOGGER.debug(() -> String.format("Fetching peer %s chain from height %d, sig %.8s", peer, commonBlockHeight, commonBlockSig58));
 
-		int ourHeight = ourInitialHeight;
 
 		// Overall plan: fetch peer's blocks first, then orphan, then apply
 
 		// Convert any leftover (post-common) block summaries into signatures to request from peer
 		List<byte[]> peerBlockSignatures = peerBlockSummaries.stream().map(BlockSummaryData::getSignature).collect(Collectors.toList());
 
+		// Keep a list of blocks received so far
+		List<Block> peerBlocks = new ArrayList<>();
+
 		// Calculate the total number of additional blocks this peer has beyond the common block
 		int additionalPeerBlocksAfterCommonBlock = peerHeight - commonBlockHeight;
 		// Subtract the number of signatures that we already have, as we don't need to request them again
 		int numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size();
 
-		// Fetch remaining block signatures, if needed
 		int retryCount = 0;
-		while (numberSignaturesRequired > 0) {
-			byte[] latestPeerSignature = peerBlockSignatures.isEmpty() ? commonBlockSig : peerBlockSignatures.get(peerBlockSignatures.size() - 1);
-			int lastPeerHeight = commonBlockHeight + peerBlockSignatures.size();
-			int numberOfSignaturesToRequest = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
+		while (height < peerHeight) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
 
-			LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
-					numberOfSignaturesToRequest, (numberOfSignaturesToRequest != 1 ? "s": ""), lastPeerHeight, Base58.encode(latestPeerSignature)));
+            // Ensure we don't request more than MAXIMUM_REQUEST_SIZE
+            int numberRequested = Math.min(numberSignaturesRequired, MAXIMUM_REQUEST_SIZE);
 
-			List<byte[]> moreBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberOfSignaturesToRequest);
+            // Do we need more signatures?
+			if (peerBlockSignatures.isEmpty() && numberRequested > 0) {
+                LOGGER.trace(String.format("Requesting %d signature%s after height %d, sig %.8s",
+                        numberRequested, (numberRequested != 1 ? "s" : ""), height, Base58.encode(latestPeerSignature)));
 
-			if (moreBlockSignatures == null || moreBlockSignatures.isEmpty()) {
-				LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
-						lastPeerHeight, Base58.encode(latestPeerSignature)));
+                peerBlockSignatures = this.getBlockSignatures(peer, latestPeerSignature, numberRequested);
+
+                if (peerBlockSignatures == null || peerBlockSignatures.isEmpty()) {
+                    LOGGER.info(String.format("Peer %s failed to respond with more block signatures after height %d, sig %.8s", peer,
+                            height, Base58.encode(latestPeerSignature)));
+
+                    // If we have already received blocks from this peer, go ahead and apply them
+                    if (peerBlocks.size() > 0) {
+                        break;
+                    }
+                    // Otherwise, give up and move on to the next peer
+                    return SynchronizationResult.NO_REPLY;
+                }
+
+                numberSignaturesRequired = peerHeight - height - peerBlockSignatures.size();
+                LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
+			}
+
+			if (peerBlockSignatures.isEmpty()) {
+				LOGGER.trace(String.format("No more signatures or blocks to request from peer %s", peer));
+				break;
+			}
+
+			byte[] nextPeerSignature = peerBlockSignatures.get(0);
+			int nextHeight = height + 1;
+
+			LOGGER.trace(String.format("Fetching block %d, sig %.8s from %s", nextHeight, Base58.encode(nextPeerSignature), peer));
+			Block newBlock = this.fetchBlock(repository, peer, nextPeerSignature);
+
+			if (newBlock == null) {
+				LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer,
+						nextHeight, Base58.encode(nextPeerSignature)));
 
 				if (retryCount >= MAXIMUM_RETRIES) {
-					// Give up with this peer
+
+					// If we have already received blocks from this peer, go ahead and apply them
+					if (peerBlocks.size() > 0) {
+						break;
+					}
+					// Otherwise, give up and move on to the next peer
 					return SynchronizationResult.NO_REPLY;
-				}
-				else {
+
+				} else {
+					// Re-fetch signatures, in case the peer is now on a different fork
+					peerBlockSignatures.clear();
+					numberSignaturesRequired = peerHeight - height;
+
 					// Retry until retryCount reaches MAXIMUM_RETRIES
 					retryCount++;
 					int triesRemaining = MAXIMUM_RETRIES - retryCount;
-					LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": "")));
+					LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s" : "")));
 					continue;
 				}
 			}
@@ -400,62 +451,31 @@ public class Synchronizer {
 			// Reset retryCount because the last request succeeded
 			retryCount = 0;
 
-			LOGGER.trace(String.format("Received %s signature%s", peerBlockSignatures.size(), (peerBlockSignatures.size() != 1 ? "s" : "")));
-
-			peerBlockSignatures.addAll(moreBlockSignatures);
-			numberSignaturesRequired = additionalPeerBlocksAfterCommonBlock - peerBlockSignatures.size();
-		}
-
-		// Fetch blocks using signatures
-		LOGGER.debug(String.format("Fetching new blocks from peer %s after height %d", peer, commonBlockHeight));
-		List<Block> peerBlocks = new ArrayList<>();
-
-		retryCount = 0;
-		while (peerBlocks.size() < peerBlockSignatures.size()) {
-			byte[] blockSignature = peerBlockSignatures.get(peerBlocks.size());
-
-			LOGGER.debug(String.format("Fetching block with signature %.8s", Base58.encode(blockSignature)));
-			int blockHeightToRequest = commonBlockHeight + peerBlocks.size() + 1; // +1 because we are requesting the next block, beyond what we already have in the peerBlocks array
-			Block newBlock = this.fetchBlock(repository, peer, blockSignature);
-
-			if (newBlock == null) {
-				LOGGER.info(String.format("Peer %s failed to respond with block for height %d, sig %.8s", peer, blockHeightToRequest, Base58.encode(blockSignature)));
-
-				if (retryCount >= MAXIMUM_RETRIES) {
-					// Give up with this peer
-					return SynchronizationResult.NO_REPLY;
-				}
-				else {
-					// Retry until retryCount reaches MAXIMUM_RETRIES
-					retryCount++;
-					int triesRemaining = MAXIMUM_RETRIES - retryCount;
-					LOGGER.info(String.format("Re-issuing request to peer %s (%d attempt%s remaining)", peer, triesRemaining, (triesRemaining != 1 ? "s": "")));
-					continue;
-				}
-			}
+			LOGGER.trace(String.format("Fetched block %d, sig %.8s from %s", nextHeight, Base58.encode(latestPeerSignature), peer));
 
 			if (!newBlock.isSignatureValid()) {
 				LOGGER.info(String.format("Peer %s sent block with invalid signature for height %d, sig %.8s", peer,
-						blockHeightToRequest, Base58.encode(blockSignature)));
+						nextHeight, Base58.encode(latestPeerSignature)));
 				return SynchronizationResult.INVALID_DATA;
 			}
 
-			// Reset retryCount because the last request succeeded
-			retryCount = 0;
-
-			LOGGER.debug(String.format("Received block with height %d, sig: %.8s", newBlock.getBlockData().getHeight(), Base58.encode(blockSignature)));
-
 			// Transactions are transmitted without approval status so determine that now
 			for (Transaction transaction : newBlock.getTransactions())
 				transaction.setInitialApprovalStatus();
 
 			peerBlocks.add(newBlock);
+
+			// Now that we've received this block, we can increase our height and move on to the next one
+			latestPeerSignature = nextPeerSignature;
+			peerBlockSignatures.remove(0);
+			++height;
 		}
 
 		// Unwind to common block (unless common block is our latest block)
-		LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s", commonBlockHeight, commonBlockSig58));
+		int ourHeight = ourInitialHeight;
+		LOGGER.debug(String.format("Orphaning blocks back to common block height %d, sig %.8s. Our height: %d", commonBlockHeight, commonBlockSig58, ourHeight));
 
-		BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourHeight);
+		BlockData orphanBlockData = repository.getBlockRepository().fromHeight(ourInitialHeight);
 		while (ourHeight > commonBlockHeight) {
 			if (Controller.isStopping())
 				return SynchronizationResult.SHUTTING_DOWN;
@@ -477,6 +497,9 @@ public class Synchronizer {
 		LOGGER.debug(String.format("Orphaned blocks back to height %d, sig %.8s - applying new blocks from peer %s", commonBlockHeight, commonBlockSig58, peer));
 
 		for (Block newBlock : peerBlocks) {
+			if (Controller.isStopping())
+				return SynchronizationResult.SHUTTING_DOWN;
+
 			ValidationResult blockResult = newBlock.isValid();
 			if (blockResult != ValidationResult.OK) {
 				LOGGER.info(String.format("Peer %s sent invalid block for height %d, sig %.8s: %s", peer,
@@ -629,6 +652,9 @@ public class Synchronizer {
 		final int firstBlockHeight = blockSummaries.get(0).getHeight();
 
 		for (int i = 0; i < blockSummaries.size(); ++i) {
+			if (Controller.isStopping())
+				return;
+
 			BlockSummaryData blockSummary = blockSummaries.get(i);
 
 			// Qortal: minter is always a reward-share, so find actual minter and get their effective minting level

tools/publish-auto-update.pl

@@ -57,9 +57,10 @@ $timestamp *= 1000; # Convert to milliseconds
 
 # locate sha256 utility
 my $SHA256 = `which sha256sum || which sha256`;
+chomp $SHA256;
 
 # SHA256 of actual update file
-my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256}`;
+my $sha256 = `git show auto-update-${commit_hash}:${project}.update | ${SHA256} | head -c 64`;
 die("Can't calculate SHA256 of ${project}.update\n") unless $sha256 =~ m/(\S{64})/;
 chomp $sha256;