Bump version to 1.5.4

When sending or requesting more than 1000 online accounts, peers would be disconnected with an EOF or connection reset error due to an intentional null response. This response has been removed and it will instead now only send the first 1000 accounts, which prevents the disconnections from occurring.

In theory, these accounts should be in a different order on each node, so the 1000 limit should still result in a fairly even propagation of accounts. However, we may want to consider increasing this limit, to maximise the propagation speed.

Thanks to szisti for tracking this one down.

.gitignore

@@ -1,6 +1,7 @@
 /db*
 /bin/
 /target/
+/qortal-backup/
 /log.txt.*
 /arbitrary*
 /Qortal-BTC*

WindowsInstaller/Qortal.aip

@@ -17,10 +17,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{75ABAC87-9F88-4229-A7D5-DE07F082566D} 1049:{42997A88-BD5D-4829-9FFD-96D0F14A2572} 2052:{0F6AAF5B-3089-4837-AC63-DEEC282862D9} 2057:{F3CAA239-F0FC-456F-B3F8-D5D846030731} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{259B155C-9CE9-4EF7-AA89-CFF2F118196B} 1049:{61A8713C-B9C5-489D-92FF-0A578697E1C2} 2052:{C59AF8F7-326D-4C43-A02D-7F4BAB56861A} 2057:{58DEB696-96AD-49DC-AD18-02B7B04FDAE3} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="1.5.2" Type="32"/>
+    <ROW Property="ProductVersion" Value="1.5.3" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -212,7 +212,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{922F3242-D791-484E-B034-2EC3EBE7E57F}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{5B3B8F5D-88BD-495F-9DD4-F749533C1F89}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>1.5.2</version>
+	<version>1.5.4</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/api/resource/CrossChainHtlcResource.java

@@ -20,10 +20,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bitcoinj.core.*;
 import org.bitcoinj.script.Script;
-import org.qortal.api.ApiError;
-import org.qortal.api.ApiErrors;
-import org.qortal.api.ApiExceptionFactory;
-import org.qortal.api.Security;
+import org.qortal.api.*;
 import org.qortal.api.model.CrossChainBitcoinyHTLCStatus;
 import org.qortal.crosschain.*;
 import org.qortal.crypto.Crypto;
@@ -279,6 +276,97 @@ public class CrossChainHtlcResource {
 		}
 	}
 
+	@GET
+	@Path("/redeemAll/LITECOIN")
+	@Operation(
+			summary = "Redeems HTLC for all applicable ATs in tradebot data",
+			description = "To be used by a QORT seller (Bob) who needs to redeem LTC proceeds that are stuck in P2SH transactions.<br>" +
+					"This requires Bob's trade bot data to be present in the database for any ATs that need redeeming.<br>" +
+					"Returns true if at least one trade is redeemed. More detail is available in the log.txt.* file.",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.TEXT_PLAIN, schema = @Schema(type = "boolean"))
+					)
+			}
+	)
+	@ApiErrors({ApiError.INVALID_CRITERIA, ApiError.INVALID_ADDRESS, ApiError.ADDRESS_UNKNOWN})
+	public boolean redeemAllHtlc() {
+		Security.checkApiCallAllowed(request);
+		boolean success = false;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			List<TradeBotData> allTradeBotData = repository.getCrossChainRepository().getAllTradeBotData();
+
+			for (TradeBotData tradeBotData : allTradeBotData) {
+				String atAddress = tradeBotData.getAtAddress();
+				if (atAddress == null) {
+					LOGGER.info("Missing AT address in tradebot data", atAddress);
+					continue;
+				}
+
+				String tradeState = tradeBotData.getState();
+				if (tradeState == null) {
+					LOGGER.info("Missing trade state for AT {}", atAddress);
+					continue;
+				}
+
+				if (tradeState.startsWith("ALICE")) {
+					LOGGER.info("AT {} isn't redeemable because it is a buy order", atAddress);
+					continue;
+				}
+
+				ATData atData = repository.getATRepository().fromATAddress(atAddress);
+				if (atData == null) {
+					LOGGER.info("Couldn't find AT with address {}", atAddress);
+					continue;
+				}
+
+				ACCT acct = SupportedBlockchain.getAcctByCodeHash(atData.getCodeHash());
+				if (acct == null) {
+					continue;
+				}
+
+				CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData);
+				if (crossChainTradeData == null) {
+					LOGGER.info("Couldn't find crosschain trade data for AT {}", atAddress);
+					continue;
+				}
+
+				// Attempt to find secret from the buyer's message to AT
+				byte[] decodedSecret = LitecoinACCTv1.findSecretA(repository, crossChainTradeData);
+				if (decodedSecret == null) {
+					LOGGER.info("Unable to find secret-A from redeem message to AT {}", atAddress);
+					continue;
+				}
+
+				// Search for the tradePrivateKey in the tradebot data
+				byte[] decodedPrivateKey = tradeBotData.getTradePrivateKey();
+
+				// Search for the litecoin receiving address PKH in the tradebot data
+				byte[] litecoinReceivingAccountInfo = tradeBotData.getReceivingAccountInfo();
+
+				try {
+					LOGGER.info("Attempting to redeem P2SH balance associated with AT {}...", atAddress);
+					boolean redeemed = this.doRedeemHtlc(atAddress, decodedPrivateKey, decodedSecret, litecoinReceivingAccountInfo);
+					if (redeemed) {
+						LOGGER.info("Redeemed P2SH balance associated with AT {}", atAddress);
+						success = true;
+					}
+					else {
+						LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Already redeemed?", atAddress);
+					}
+				} catch (ApiException e) {
+					LOGGER.info("Couldn't redeem P2SH balance associated with AT {}. Missing data?", atAddress);
+				}
+			}
+
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
+
+		return success;
+	}
+
 	private boolean doRedeemHtlc(String atAddress, byte[] decodedTradePrivateKey, byte[] decodedSecret, byte[] litecoinReceivingAccountInfo) {
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			ATData atData = repository.getATRepository().fromATAddress(atAddress);

src/main/java/org/qortal/api/websocket/ChatMessagesWebSocket.java

@@ -115,6 +115,9 @@ public class ChatMessagesWebSocket extends ApiWebSocket {
 	}
 
 	private void onNotify(Session session, ChatTransactionData chatTransactionData, List<String> involvingAddresses) {
+		if (chatTransactionData == null)
+			return;
+
 		// We only want direct/non-group messages where sender/recipient match our addresses
 		String recipient = chatTransactionData.getRecipient();
 		if (recipient == null)

src/main/java/org/qortal/controller/Controller.java

@@ -259,17 +259,29 @@ public class Controller extends Thread {
 			throw new RuntimeException("Can't read build.properties resource", e);
 		}
 
+		// Determine build timestamp
 		String buildTimestampProperty = properties.getProperty("build.timestamp");
-		if (buildTimestampProperty == null)
+		if (buildTimestampProperty == null) {
 			throw new RuntimeException("Can't read build.timestamp from build.properties resource");
-
-		this.buildTimestamp = LocalDateTime.parse(buildTimestampProperty, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")).toEpochSecond(ZoneOffset.UTC);
+		}
+		if (buildTimestampProperty.startsWith("$")) {
+			// Maven vars haven't been replaced - this was most likely built using an IDE, not via mvn package
+			this.buildTimestamp = System.currentTimeMillis();
+			buildTimestampProperty = "unknown";
+		} else {
+			this.buildTimestamp = LocalDateTime.parse(buildTimestampProperty, DateTimeFormatter.ofPattern("yyyyMMddHHmmss")).toEpochSecond(ZoneOffset.UTC);
+		}
 		LOGGER.info(String.format("Build timestamp: %s", buildTimestampProperty));
 
+		// Determine build version
 		String buildVersionProperty = properties.getProperty("build.version");
-		if (buildVersionProperty == null)
+		if (buildVersionProperty == null) {
 			throw new RuntimeException("Can't read build.version from build.properties resource");
-
+		}
+		if (buildVersionProperty.contains("${git.commit.id.abbrev}")) {
+			// Maven vars haven't been replaced - this was most likely built using an IDE, not via mvn package
+			buildVersionProperty = buildVersionProperty.replace("${git.commit.id.abbrev}", "debug");
+		}
 		this.buildVersion = VERSION_PREFIX + buildVersionProperty;
 		LOGGER.info(String.format("Build version: %s", this.buildVersion));
 
@@ -678,7 +690,7 @@ public class Controller extends Thread {
 
 		final int peersRemoved = peersBeforeComparison - peers.size();
 		if (peersRemoved > 0 && peers.size() > 0)
-			LOGGER.info(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
+			LOGGER.debug(String.format("Ignoring %d peers on inferior chains. Peers remaining: %d", peersRemoved, peers.size()));
 
 		if (peers.isEmpty())
 			return;
@@ -687,7 +699,7 @@ public class Controller extends Thread {
 			StringBuilder finalPeersString = new StringBuilder();
 			for (Peer peer : peers)
 				finalPeersString = finalPeersString.length() > 0 ? finalPeersString.append(", ").append(peer) : finalPeersString.append(peer);
-			LOGGER.info(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
+			LOGGER.debug(String.format("Choosing random peer from: [%s]", finalPeersString.toString()));
 		}
 
 		// Pick random peer to sync with
@@ -883,14 +895,19 @@ public class Controller extends Thread {
 
 			List<TransactionData> transactions = repository.getTransactionRepository().getUnconfirmedTransactions();
 
+			int deletedCount = 0;
 			for (TransactionData transactionData : transactions) {
 				Transaction transaction = Transaction.fromData(repository, transactionData);
 
 				if (now >= transaction.getDeadline()) {
-					LOGGER.info(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
+					LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
 					repository.getTransactionRepository().delete(transactionData);
+					deletedCount++;
 				}
 			}
+			if (deletedCount > 0) {
+				LOGGER.info(String.format("Deleted %d expired, unconfirmed transaction%s", deletedCount, (deletedCount == 1 ? "" : "s")));
+			}
 
 			repository.saveChanges();
 		} catch (DataException e) {

src/main/java/org/qortal/controller/Synchronizer.java

@@ -139,7 +139,7 @@ public class Synchronizer {
 
 				if (wereNewRequestsMade) {
 					final long totalTimeTaken = System.currentTimeMillis() - startTime;
-					LOGGER.info(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken));
+					LOGGER.debug(String.format("Finished searching for common blocks with %d peer%s. Found: %d. Total time taken: %d ms", peers.size(), (peers.size() != 1 ? "s" : ""), commonBlocksFound, totalTimeTaken));
 				}
 
 				return SynchronizationResult.OK;

src/main/java/org/qortal/controller/tradebot/BitcoinACCTv1TradeBot.java

@@ -383,15 +383,6 @@ public class BitcoinACCTv1TradeBot implements AcctTradeBot {
 			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
 			if (atData == null) {
 				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
-
-				// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
-				// and so wipe the trade-bot entry
-				if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
-					LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
-					repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-					repository.saveChanges();
-				}
-
 				return;
 			}
 

src/main/java/org/qortal/controller/tradebot/LitecoinACCTv1TradeBot.java

@@ -387,15 +387,6 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 			atData = repository.getATRepository().fromATAddress(tradeBotData.getAtAddress());
 			if (atData == null) {
 				LOGGER.debug(() -> String.format("Unable to fetch trade AT %s from repository", tradeBotData.getAtAddress()));
-
-				// If it has been over 24 hours since we last updated this trade-bot entry then assume AT is never coming back
-				// and so wipe the trade-bot entry
-				if (tradeBotData.getTimestamp() + MAX_AT_CONFIRMATION_PERIOD < NTP.getTime()) {
-					LOGGER.info(() -> String.format("AT %s has been gone for too long - deleting trade-bot entry", tradeBotData.getAtAddress()));
-					repository.getCrossChainRepository().delete(tradeBotData.getTradePrivateKey());
-					repository.saveChanges();
-				}
-
 				return;
 			}
 
@@ -725,9 +716,9 @@ public class LitecoinACCTv1TradeBot implements AcctTradeBot {
 			// Not finished yet
 			return;
 
-		// If AT is not REDEEMED then something has gone wrong
-		if (crossChainTradeData.mode != AcctMode.REDEEMED) {
-			// Not redeemed so must be refunded/cancelled
+		// If AT is REFUNDED or CANCELLED then something has gone wrong
+		if (crossChainTradeData.mode == AcctMode.REFUNDED || crossChainTradeData.mode == AcctMode.CANCELLED) {
+			// Alice hasn't redeemed the QORT, so there is no point in trying to redeem the LTC
 			TradeBot.updateTradeBotState(repository, tradeBotData, State.BOB_REFUNDED,
 					() -> String.format("AT %s has auto-refunded - trade aborted", tradeBotData.getAtAddress()));
 

src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java

@@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.qortal.data.network.OnlineAccountData;
 import org.qortal.transform.Transformer;
@@ -25,7 +26,7 @@ public class GetOnlineAccountsMessage extends Message {
 	private GetOnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
 		super(id, MessageType.GET_ONLINE_ACCOUNTS);
 
-		this.onlineAccounts = onlineAccounts;
+		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
 	}
 
 	public List<OnlineAccountData> getOnlineAccounts() {
@@ -35,12 +36,9 @@ public class GetOnlineAccountsMessage extends Message {
 	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
 		final int accountCount = bytes.getInt();
 
-		if (accountCount > MAX_ACCOUNT_COUNT)
-			return null;
-
 		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
 
-		for (int i = 0; i < accountCount; ++i) {
+		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
 			long timestamp = bytes.getLong();
 
 			byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];

src/main/java/org/qortal/network/message/OnlineAccountsMessage.java

@@ -6,6 +6,7 @@ import java.io.UnsupportedEncodingException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import org.qortal.data.network.OnlineAccountData;
 import org.qortal.transform.Transformer;
@@ -25,7 +26,7 @@ public class OnlineAccountsMessage extends Message {
 	private OnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
 		super(id, MessageType.ONLINE_ACCOUNTS);
 
-		this.onlineAccounts = onlineAccounts;
+		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
 	}
 
 	public List<OnlineAccountData> getOnlineAccounts() {
@@ -35,12 +36,9 @@ public class OnlineAccountsMessage extends Message {
 	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
 		final int accountCount = bytes.getInt();
 
-		if (accountCount > MAX_ACCOUNT_COUNT)
-			return null;
-
 		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
 
-		for (int i = 0; i < accountCount; ++i) {
+		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
 			long timestamp = bytes.getLong();
 
 			byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];