Added debug logging of new file list stats.

These params are optional and the process will function without them, just less efficiently.

TestNets.md

@@ -41,13 +41,39 @@
 - Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead)
 - Probably best to perform API call `DELETE /peers/known`
 - Add other nodes via API call `POST /peers <peer-hostname-or-IP>`
-- Add minting private key to node(s) via API call `POST /admin/mintingaccounts <minting-private-key>`
-	This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block
+- Add minting private key to nodes via API call `POST /admin/mintingaccounts <minting-private-key>`
+    The keys must have corresponding `REWARD_SHARE` transactions in testnet genesis block
+- You must have at least 2 separate minting keys and two separate nodes. Assign one minting key to each node.
+- Alternatively, comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java to allow for a single node and key.
 - Wait for genesis block timestamp to pass
 - A node should mint block 2 approximately 60 seconds after genesis block timestamp
 - Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain`
 - You can also use API call `POST /admin/forcesync <connected-peer-IP-and-port>` on stuck nodes
 
+## Single-node testnet
+
+A single-node testnet is possible with code modifications, for basic testing, or to more easily start a new testnet.
+To do so, follow these steps:
+- Comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java
+- Comment out the `minBlockchainPeers` validation in Settings.validate()
+- Set `minBlockchainPeers` to 0 in settings.json
+- Set `Synchronizer.RECOVERY_MODE_TIMEOUT` to `0`
+- All other steps should remain the same. Only a single reward share key is needed.
+- Remember to put these values back after introducing other nodes
+
+## Fixed network
+
+To restrict a testnet to a set of private nodes, you can use the "fixed network" feature.
+This ensures that the testnet nodes only communicate with each other and not other known peers.
+To do this, add the following setting to each testnet node, substituting the IP addresses:
+```
+"fixedNetwork": [
+  "192.168.0.101:62392",
+  "192.168.0.102:62392",
+  "192.168.0.103:62392"
+]
+```
+
 ## Dealing with stuck chain
 
 Maybe your nodes have been offline and no-one has minted a recent testnet block.

WindowsInstaller/Qortal.aip

@@ -17,10 +17,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{8CB28EE4-A407-4CED-A7C5-5138BCBF4B13} 1049:{C416F381-0C39-459C-A571-AC686D57B808} 2052:{A3DBA289-D2F0-4A3F-94DC-B89F5A6465DE} 2057:{893B13A5-45B6-4E49-8690-3FE46BCA50DF} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{5FC8DCC3-BF9C-4D72-8C6D-940340ACD1B8} 1049:{1DEF14AB-2397-4517-B3C8-13221B921753} 2052:{B9E3C1DF-C92D-440A-9A21-869582F8585F} 2057:{91D69E7B-CA7D-4449-8E8A-F22DCEA546FC} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="3.1.0" Type="32"/>
+    <ROW Property="ProductVersion" Value="3.1.1" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -212,7 +212,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{551D2FA5-BE83-4DA1-A96E-7D15C517F9FF}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{42F5EC19-E46F-4299-B9F7-6E1112F6E4FB}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>

lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.dosse</groupId>
+  <artifactId>WaifUPnP</artifactId>
+  <version>1.1</version>
+  <description>POM was created from install:install-file</description>
+</project>

lib/com/dosse/WaifUPnP/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>com.dosse</groupId>
+  <artifactId>WaifUPnP</artifactId>
+  <versioning>
+    <release>1.1</release>
+    <versions>
+      <version>1.1</version>
+    </versions>
+    <lastUpdated>20220218200127</lastUpdated>
+  </versioning>
+</metadata>

pom.xml

@@ -21,6 +21,7 @@
 		<dagger.version>1.2.2</dagger.version>
 		<guava.version>28.1-jre</guava.version>
 		<hsqldb.version>2.5.1</hsqldb.version>
+		<upnp.version>1.1</upnp.version>
 		<jersey.version>2.29.1</jersey.version>
 		<jetty.version>9.4.29.v20200521</jetty.version>
 		<log4j.version>2.17.1</log4j.version>
@@ -427,6 +428,12 @@
 			<artifactId>AT</artifactId>
 			<version>${ciyam-at.version}</version>
 		</dependency>
+		<!-- UPnP support -->
+		<dependency>
+			<groupId>com.dosse</groupId>
+			<artifactId>WaifUPnP</artifactId>
+			<version>${upnp.version}</version>
+		</dependency>
 		<!-- Bitcoin support -->
 		<dependency>
 			<groupId>org.bitcoinj</groupId>

src/main/java/org/qortal/account/Account.java

@@ -272,7 +272,7 @@ public class Account {
 	/**
 	 * Returns 'effective' minting level, or zero if reward-share does not exist.
 	 * <p>
-	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
+	 * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call
 	 * 
 	 * @param repository
 	 * @param rewardSharePublicKey
@@ -288,5 +288,26 @@ public class Account {
 		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
 		return rewardShareMinter.getEffectiveMintingLevel();
 	}
+	/**
+	 * Returns 'effective' minting level, with a fix for the zero level.
+	 * <p>
+	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
+	 *
+	 * @param repository
+	 * @param rewardSharePublicKey
+	 * @return 0+
+	 * @throws DataException
+	 */
+	public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
+		// Find actual minter and get their effective minting level
+		RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
+		if (rewardShareData == null)
+			return 0;
 
+		else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship
+			return 0;
+
+		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
+		return rewardShareMinter.getEffectiveMintingLevel();
+	}
 }

src/main/java/org/qortal/api/HTMLParser.java

@@ -28,6 +28,11 @@ public class HTMLParser {
             // Add base href tag
             String baseElement = String.format("<base href=\"%s\">", baseUrl);
             head.get(0).prepend(baseElement);
+
+            // Add meta charset tag
+            String metaCharsetElement = "<meta charset=\"UTF-8\">";
+            head.get(0).prepend(metaCharsetElement);
+
         }
         String html = document.html();
         this.data = html.getBytes();

src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java

@@ -17,7 +17,7 @@ import java.util.Map;
 
 
 @Path("/")
-@Tag(name = "Gateway")
+@Tag(name = "Domain Map")
 public class DomainMapResource {
 
     @Context HttpServletRequest request;

src/main/java/org/qortal/api/resource/AddressesResource.java

@@ -198,7 +198,7 @@ public class AddressesResource {
 
 			for (OnlineAccountData onlineAccountData : onlineAccounts) {
 				try {
-					final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey());
+					final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey());
 
 					OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream()
 							.filter(a -> a.getLevel() == minterLevel)

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -315,6 +315,7 @@ public class AdminResource {
 
 			repository.getAccountRepository().save(mintingAccountData);
 			repository.saveChanges();
+			repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup  MintingAccounts.json 
 		} catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
 		} catch (DataException e) {
@@ -355,6 +356,7 @@ public class AdminResource {
 				return "false";
 
 			repository.saveChanges();
+			repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup  MintingAccounts.json 
 		} catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
 		} catch (DataException e) {
@@ -546,7 +548,7 @@ public class AdminResource {
 	@Path("/repository/data")
 	@Operation(
 		summary = "Export sensitive/node-local data from repository.",
-		description = "Exports data to .script files on local machine"
+		description = "Exports data to .json files on local machine"
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")

src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java

@@ -122,7 +122,7 @@ public class CrossChainBitcoinResource {
 	@Path("/send")
 	@Operation(
 		summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address",
-		description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
+		description = "Currently supports 'legacy' P2PKH Bitcoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(

src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java

@@ -122,7 +122,7 @@ public class CrossChainLitecoinResource {
 	@Path("/send")
 	@Operation(
 		summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address",
-		description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
+		description = "Currently supports 'legacy' P2PKH Litecoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(

src/main/java/org/qortal/api/resource/PeersResource.java

@@ -354,7 +354,7 @@ public class PeersResource {
 
 		List<Peer> connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList());
 		for (Peer peer : connectedPeers) {
-			if (peer.isOutbound()) {
+			if (!peer.isOutbound()) {
 				peersSummary.inboundConnections++;
 			}
 			else {

src/main/java/org/qortal/api/resource/TransactionsResource.java

@@ -638,7 +638,10 @@ public class TransactionsResource {
 		ApiError.BLOCKCHAIN_NEEDS_SYNC, ApiError.INVALID_SIGNATURE, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE
 	})
 	public String processTransaction(String rawBytes58) {
-		if (!Controller.getInstance().isUpToDate())
+		// Only allow a transaction to be processed if our latest block is less than 30 minutes old
+		// If older than this, we should first wait until the blockchain is synced
+		final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L);
+		if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
 
 		byte[] rawBytes = Base58.decode(rawBytes58);

src/main/java/org/qortal/api/websocket/PresenceWebSocket.java

@@ -20,6 +20,7 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
 import org.eclipse.jetty.websocket.api.annotations.WebSocket;
 import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.transaction.PresenceTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -99,13 +100,13 @@ public class PresenceWebSocket extends ApiWebSocket implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		// We use NewBlockEvent as a proxy for 1-minute timer
-		if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent))
+		// We use Synchronizer.NewChainTipEvent as a proxy for 1-minute timer
+		if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
 		removeOldEntries();
 
-		if (event instanceof Controller.NewBlockEvent)
+		if (event instanceof Synchronizer.NewChainTipEvent)
 			// We only wanted a chance to cull old entries
 			return;
 

src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java

@@ -23,6 +23,7 @@ import org.eclipse.jetty.websocket.api.annotations.WebSocket;
 import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
 import org.qortal.api.model.CrossChainOfferSummary;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.crosschain.SupportedBlockchain;
 import org.qortal.crosschain.ACCT;
 import org.qortal.crosschain.AcctMode;
@@ -80,10 +81,10 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		if (!(event instanceof Controller.NewBlockEvent))
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
-		BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData();
+		BlockData blockData = ((Synchronizer.NewChainTipEvent) event).getNewChainTip();
 
 		// Process any new info
 

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -366,6 +366,21 @@ public class ArbitraryDataFile {
         return false;
     }
 
+    public boolean delete(int attempts) {
+        // Keep trying to delete the data until it is deleted, or we reach 10 attempts
+        for (int i=0; i<attempts; i++) {
+            if (this.delete()) {
+                return true;
+            }
+            try {
+                Thread.sleep(1000L);
+            } catch (InterruptedException e) {
+                // Fall through to exit method
+            }
+        }
+        return false;
+    }
+
     public boolean deleteAllChunks() {
         boolean success = false;
 

src/main/java/org/qortal/controller/Controller.java

@@ -105,6 +105,8 @@ public class Controller extends Thread {
 	private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
 	/** How many (latest) blocks' worth of online accounts we cache */
 	private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
+	private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
+
 
 	private static volatile boolean isStopping = false;
 	private static BlockMinter blockMinter = null;
@@ -765,14 +767,14 @@ public class Controller extends Thread {
 				actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED");
 				SysTray.getInstance().setTrayIcon(2);
 			}
-			else if (Synchronizer.getInstance().isSynchronizing()) {
-				actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
-				SysTray.getInstance().setTrayIcon(3);
-			}
 			else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) {
 				actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING");
 				SysTray.getInstance().setTrayIcon(3);
 			}
+			else if (!this.isUpToDate()) {
+				actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
+				SysTray.getInstance().setTrayIcon(3);
+			}
 			else {
 				actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED");
 				SysTray.getInstance().setTrayIcon(4);
@@ -1282,6 +1284,14 @@ public class Controller extends Thread {
 				onNetworkOnlineAccountsMessage(peer, message);
 				break;
 
+			case GET_ONLINE_ACCOUNTS_V2:
+				onNetworkGetOnlineAccountsV2Message(peer, message);
+				break;
+
+			case ONLINE_ACCOUNTS_V2:
+				onNetworkOnlineAccountsV2Message(peer, message);
+				break;
+
 			case GET_ARBITRARY_DATA:
 				// Not currently supported
 				break;
@@ -1687,6 +1697,53 @@ public class Controller extends Thread {
 		}
 	}
 
+	private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
+		GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message;
+
+		List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
+
+		// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
+		List<OnlineAccountData> accountsToSend;
+		synchronized (this.onlineAccounts) {
+			accountsToSend = new ArrayList<>(this.onlineAccounts);
+		}
+
+		Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
+
+		SEND_ITERATOR:
+		while (iterator.hasNext()) {
+			OnlineAccountData onlineAccountData = iterator.next();
+
+			for (int i = 0; i < excludeAccounts.size(); ++i) {
+				OnlineAccountData excludeAccountData = excludeAccounts.get(i);
+
+				if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
+					iterator.remove();
+					continue SEND_ITERATOR;
+				}
+			}
+		}
+
+		Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
+		peer.sendMessage(onlineAccountsMessage);
+
+		LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
+	}
+
+	private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) {
+		OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message;
+
+		List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
+		LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
+				this.verifyAndAddAccount(repository, onlineAccountData);
+		} catch (DataException e) {
+			LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
+		}
+	}
+
 	// Utilities
 
 	private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
@@ -1798,11 +1855,17 @@ public class Controller extends Thread {
 
 		// Request data from other peers?
 		if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
-			Message message;
+			List<OnlineAccountData> safeOnlineAccounts;
 			synchronized (this.onlineAccounts) {
-				message = new GetOnlineAccountsMessage(this.onlineAccounts);
+				safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
 			}
-			Network.getInstance().broadcast(peer -> message);
+
+			Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
+			Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
+
+			Network.getInstance().broadcast(peer ->
+				peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
+			);
 		}
 
 		// Refresh our online accounts signatures?
@@ -1894,8 +1957,12 @@ public class Controller extends Thread {
 			if (!hasInfoChanged)
 				return;
 
-			Message message = new OnlineAccountsMessage(ourOnlineAccounts);
-			Network.getInstance().broadcast(peer -> message);
+		Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
+		Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
+
+		Network.getInstance().broadcast(peer ->
+				peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
+		);
 
 			LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
 		}
@@ -1981,10 +2048,13 @@ public class Controller extends Thread {
 		return peers;
 	}
 
-	/** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */
-	public boolean isUpToDate() {
+	/**
+	 * Returns whether we think our node has up-to-date blockchain based on our info about other peers.
+	 * @param minLatestBlockTimestamp - the minimum block timestamp to be considered recent
+	 * @return boolean - whether our node's blockchain is up to date or not
+	 */
+	public boolean isUpToDate(Long minLatestBlockTimestamp) {
 		// Do we even have a vaguely recent block?
-		final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
 		if (minLatestBlockTimestamp == null)
 			return false;
 
@@ -2010,6 +2080,16 @@ public class Controller extends Thread {
 		return !peers.isEmpty();
 	}
 
+	/**
+	 * Returns whether we think our node has up-to-date blockchain based on our info about other peers.
+	 * Uses the default minLatestBlockTimestamp value.
+	 * @return boolean - whether our node's blockchain is up to date or not
+	 */
+	public boolean isUpToDate() {
+		final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
+		return this.isUpToDate(minLatestBlockTimestamp);
+	}
+
 	/** Returns minimum block timestamp for block to be considered 'recent', or <tt>null</tt> if NTP not synced. */
 	public static Long getMinimumLatestBlockTimestamp() {
 		Long now = NTP.getTime();

src/main/java/org/qortal/controller/Synchronizer.java

@@ -22,6 +22,8 @@ import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.message.BlockMessage;
@@ -96,6 +98,24 @@ public class Synchronizer extends Thread {
 		OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE, SHUTTING_DOWN;
 	}
 
+	public static class NewChainTipEvent implements Event {
+		private final BlockData priorChainTip;
+		private final BlockData newChainTip;
+
+		public NewChainTipEvent(BlockData priorChainTip, BlockData newChainTip) {
+			this.priorChainTip = priorChainTip;
+			this.newChainTip = newChainTip;
+		}
+
+		public BlockData getPriorChainTip() {
+			return this.priorChainTip;
+		}
+
+		public BlockData getNewChainTip() {
+			return this.newChainTip;
+		}
+	}
+
 	// Constructors
 
 	private Synchronizer() {
@@ -338,6 +358,8 @@ public class Synchronizer extends Thread {
 
 				Network network = Network.getInstance();
 				network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
+
+				EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip));
 			}
 
 			return syncResult;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -222,7 +222,11 @@ public class ArbitraryDataCleanupManager extends Thread {
 				try (final Repository repository = RepositoryManager.getRepository()) {
 
 					// Check if there are any hosted files that don't have matching transactions
-					this.checkForExpiredTransactions(repository);
+					// UPDATE: This has been disabled for now as it was deleting valid transactions
+					// and causing chunks to go missing on the network. If ever re-enabled, we MUST
+					// ensure that original copies of data aren't deleted, and that sufficient time
+					// is allowed (ideally several hours) before treating a transaction as missing.
+					// this.checkForExpiredTransactions(repository);
 
 					// Delete additional data at random if we're over our storage limit
 					// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -29,6 +29,7 @@ public class ArbitraryDataFileListManager {
 
     private static ArbitraryDataFileListManager instance;
 
+    private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0";
 
     /**
      * Map of recent incoming requests for ARBITRARY transaction data file lists.
@@ -403,6 +404,13 @@ public class ArbitraryDataFileListManager {
         ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
         LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
 
+        if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
+            long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
+            LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
+                    totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
+                    arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+        }
+
         // Do we have a pending request for this data?
         Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
         if (request == null || request.getA() == null) {
@@ -472,12 +480,26 @@ public class ArbitraryDataFileListManager {
             if (!isBlocked) {
                 Peer requestingPeer = request.getB();
                 if (requestingPeer != null) {
+                    Long requestTime = arbitraryDataFileListMessage.getRequestTime();
+                    Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
+
                     // Add each hash to our local mapping so we know who to ask later
                     Long now = NTP.getTime();
                     for (byte[] hash : hashes) {
                         String hash58 = Base58.encode(hash);
-                        ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
-                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
+                        ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
+                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
+                    }
+
+                    // Bump requestHops if it exists
+                    if (requestHops != null) {
+                        arbitraryDataFileListMessage.setRequestHops(++requestHops);
+                    }
+
+                    // Remove optional parameters if the requesting peer doesn't support it yet
+                    // A message with less statistical data is better than no message at all
+                    if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                        arbitraryDataFileListMessage.removeOptionalStats();
                     }
 
                     // Forward to requesting peer
@@ -582,8 +604,17 @@ public class ArbitraryDataFileListManager {
                 arbitraryDataFileListRequests.put(message.getId(), newEntry);
             }
 
-            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            String ourAddress = Network.getInstance().getOurExternalIpAddress();
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
+                    hashes, NTP.getTime(), 0, ourAddress, true);
             arbitraryDataFileListMessage.setId(message.getId());
+
+            // Remove optional parameters if the requesting peer doesn't support it yet
+            // A message with less statistical data is better than no message at all
+            if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                arbitraryDataFileListMessage.removeOptionalStats();
+            }
+
             if (!peer.sendMessage(arbitraryDataFileListMessage)) {
                 LOGGER.debug("Couldn't send list of hashes");
                 peer.disconnect("failed to send list of hashes");

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -37,7 +37,7 @@ public class ArbitraryDataFileManager extends Thread {
     /**
      * Map to keep track of our in progress (outgoing) arbitrary data file requests
      */
-    private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
+    public Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
 
     /**
      * Map to keep track of hashes that we might need to relay
@@ -148,7 +148,7 @@ public class ArbitraryDataFileManager extends Thread {
                     }
                 }
                 else {
-                    LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
+                    LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
                 }
             }
             else {
@@ -240,16 +240,7 @@ public class ArbitraryDataFileManager extends Thread {
                 ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
 
                 // Keep trying to delete the data until it is deleted, or we reach 10 attempts
-                for (int i=0; i<10; i++) {
-                    if (dataFile.delete()) {
-                        break;
-                    }
-                    try {
-                        Thread.sleep(1000L);
-                    } catch (InterruptedException e) {
-                        // Fall through to exit method
-                    }
-                }
+                dataFile.delete(10);
             }
         }
 
@@ -401,6 +392,33 @@ public class ArbitraryDataFileManager extends Thread {
         }
     }
 
+    private ArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching relay info for hash: {}", hash58);
+        List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Remove any with null requestHops
+            relayInfoList.removeIf(r -> r.getRequestHops() == null);
+
+            // If list is now empty, then just return one at random
+            if (relayInfoList.isEmpty()) {
+                return this.getRandomRelayInfoEntryForHash(hash58);
+            }
+
+            // Sort by number of hops (lowest first)
+            relayInfoList.sort(Comparator.comparingInt(ArbitraryRelayInfo::getRequestHops));
+
+            // FUTURE: secondary sort by requestTime?
+
+            ArbitraryRelayInfo relayInfo = relayInfoList.get(0);
+
+            LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops());
+            return relayInfo;
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
     private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
         LOGGER.trace("Fetching random relay info for hash: {}", hash58);
         List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
@@ -451,7 +469,7 @@ public class ArbitraryDataFileManager extends Thread {
 
         try {
             ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
-            ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
+            ArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58);
 
             if (arbitraryDataFile.exists()) {
                 LOGGER.trace("Hash {} exists", hash58);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -80,6 +80,12 @@ public class ArbitraryDataFileRequestThread implements Runnable {
                     continue;
                 }
 
+                // Skip if already requesting, but don't remove, as we might want to retry later
+                if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                    // Already requesting - leave this attempt for later
+                    continue;
+                }
+
                 // We want to process this file
                 shouldProcess = true;
                 iterator.remove();

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -16,6 +16,7 @@ import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -322,7 +323,7 @@ public class ArbitraryDataStorageManager extends Thread {
                             && path.getFileName().toString().length() > 32)
                     .collect(Collectors.toList());
         }
-        catch (IOException e) {
+        catch (IOException | UncheckedIOException e) {
             LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
         }
 

src/main/java/org/qortal/controller/tradebot/TradeBot.java

@@ -16,6 +16,7 @@ import org.bitcoinj.core.ECKey;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.model.crosschain.TradeBotCreateRequest;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
 import org.qortal.crosschain.*;
 import org.qortal.data.at.ATData;
@@ -213,7 +214,7 @@ public class TradeBot implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		if (!(event instanceof Controller.NewBlockEvent))
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
 		synchronized (this) {

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -58,9 +58,14 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	 * i.e. keys with transactions but with no unspent outputs. */
 	protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
 
-	/** How many bitcoinj wallet keys to generate in each batch. */
+	/** How many wallet keys to generate in each batch. */
 	private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3;
 
+	/** How many wallet keys to generate when using bitcoinj as the data provider.
+	 * We must use a higher value here since we are unable to request multiple batches of keys.
+	 * Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */
+	private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50;
+
 	/** Byte offset into raw block headers to block timestamp. */
 	private static final int TIMESTAMP_OFFSET = 4 + 32 + 32;
 
@@ -99,8 +104,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		try {
 			ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType();
 
-			return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH;
+			return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH || addressType == ScriptType.P2WPKH;
 		} catch (AddressFormatException e) {
+			LOGGER.error(String.format("Unrecognised address format: %s", address));
 			return false;
 		}
 	}
@@ -471,6 +477,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		List<SimpleTransaction.Output> outputs = new ArrayList<>();
 
 		boolean anyOutputAddressInWallet = false;
+		boolean transactionInvolvesExternalWallet = false;
 
 		for (BitcoinyTransaction.Input input : t.inputs) {
 			try {
@@ -485,6 +492,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 							total += inputAmount;
 							addressInWallet = true;
 						}
+						else {
+							transactionInvolvesExternalWallet = true;
+						}
 						inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
 					}
 				}
@@ -498,14 +508,17 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 					for (String address : output.addresses) {
 						boolean addressInWallet = false;
 						if (keySet.contains(address)) {
-							if (total > 0L) {
+							if (total > 0L) { // Change returned from sent amount
 								amount -= (total - output.value);
-							} else {
+							} else { // Amount received
 								amount += output.value;
 							}
 							addressInWallet = true;
 							anyOutputAddressInWallet = true;
 						}
+						else {
+							transactionInvolvesExternalWallet = true;
+						}
 						outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
 					}
 				}
@@ -520,6 +533,10 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 				amount = total * -1;
 			}
 		}
+		else if (!transactionInvolvesExternalWallet) {
+			// All inputs and outputs relate to this wallet, so the balance should be unaffected
+			amount = 0;
+		}
 		return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs);
 	}
 
@@ -612,7 +629,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 			this.keyChain = this.wallet.getActiveKeyChain();
 
 			// Set up wallet's key chain
-			this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
+			this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ);
 			this.keyChain.maybeLookAhead();
 		}
 

src/main/java/org/qortal/crosschain/ElectrumX.java

@@ -401,13 +401,36 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 				String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex");
 				long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue();
 
-				// address too, if present
+				// address too, if present in the "addresses" array
 				List<String> addresses = null;
 				Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses");
 				if (addressesObj instanceof JSONArray) {
 					addresses = new ArrayList<>();
-					for (Object addressObj : (JSONArray) addressesObj)
+					for (Object addressObj : (JSONArray) addressesObj) {
 						addresses.add((String) addressObj);
+					}
+				}
+
+				// some peers return a single "address" string
+				Object addressObj = ((JSONObject) outputJson.get("scriptPubKey")).get("address");
+				if (addressObj instanceof String) {
+					if (addresses == null) {
+						addresses = new ArrayList<>();
+					}
+					addresses.add((String) addressObj);
+				}
+
+				// For the purposes of Qortal we require all outputs to contain addresses
+				// Some servers omit this info, causing problems down the line with balance calculations
+				// Update: it turns out that they were just using a different key - "address" instead of "addresses"
+				// The code below can remain in place, just in case a peer returns a missing address in the future
+				if (addresses == null || addresses.isEmpty()) {
+					if (this.currentServer != null) {
+						this.uselessServers.add(this.currentServer);
+						this.closeServer(this.currentServer);
+					}
+					LOGGER.info("No output addresses returned for transaction {}", txHash);
+					throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash));
 				}
 
 				outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses));

src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java

@@ -9,12 +9,16 @@ public class ArbitraryRelayInfo {
     private final String signature58;
     private final Peer peer;
     private final Long timestamp;
+    private final Long requestTime;
+    private final Integer requestHops;
 
-    public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) {
+    public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) {
         this.hash58 = hash58;
         this.signature58 = signature58;
         this.peer = peer;
         this.timestamp = timestamp;
+        this.requestTime = requestTime;
+        this.requestHops = requestHops;
     }
 
     public boolean isValid() {
@@ -38,6 +42,14 @@ public class ArbitraryRelayInfo {
         return timestamp;
     }
 
+    public Long getRequestTime() {
+        return this.requestTime;
+    }
+
+    public Integer getRequestHops() {
+        return this.requestHops;
+    }
+
     @Override
     public String toString() {
         return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);

src/main/java/org/qortal/network/Handshake.java

@@ -74,6 +74,12 @@ public enum Handshake {
 			peer.setPeersConnectionTimestamp(peersConnectionTimestamp);
 			peer.setPeersVersion(versionString, version);
 
+			// Ensure the peer is running at least the version specified in MIN_PEER_VERSION
+			if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) {
+				LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString));
+				return null;
+			}
+
 			if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) {
 				// Ensure the peer is running at least the minimum version allowed for connections
 				final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
@@ -258,6 +264,9 @@ public enum Handshake {
 
 	private static final long PEER_VERSION_131 = 0x0100030001L;
 
+	/** Minimum peer version that we are allowed to communicate with */
+	private static final String MIN_PEER_VERSION = "3.1.0";
+
 	private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes
 	private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits
 	// Can always be made harder in the future...

src/main/java/org/qortal/network/Network.java

@@ -1,5 +1,6 @@
 package org.qortal.network;
 
+import com.dosse.upnp.UPnP;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
@@ -7,7 +8,6 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
 import org.qortal.controller.arbitrary.ArbitraryDataFileListManager;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.network.PeerData;
@@ -183,6 +183,14 @@ public class Network {
             }
         }
 
+        // Attempt to set up UPnP. All errors are ignored.
+        if (Settings.getInstance().isUPnPEnabled()) {
+            UPnP.openPortTCP(Settings.getInstance().getListenPort());
+        }
+        else {
+            UPnP.closePortTCP(Settings.getInstance().getListenPort());
+        }
+
         // Start up first networking thread
         networkEPC.start();
     }
@@ -243,12 +251,15 @@ public class Network {
     public boolean requestDataFromPeer(String peerAddressString, byte[] signature) {
         if (peerAddressString != null) {
             PeerAddress peerAddress = PeerAddress.fromString(peerAddressString);
+            PeerData peerData = null;
 
             // Reuse an existing PeerData instance if it's already in the known peers list
-            PeerData peerData = this.allKnownPeers.stream()
-                    .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
-                    .findFirst()
-                    .orElse(null);
+            synchronized (this.allKnownPeers) {
+                peerData = this.allKnownPeers.stream()
+                        .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
+                        .findFirst()
+                        .orElse(null);
+            }
 
             if (peerData == null) {
                 // Not a known peer, so we need to create one
@@ -263,10 +274,13 @@ public class Network {
             }
 
             // Check if we're already connected to and handshaked with this peer
-            Peer connectedPeer = this.connectedPeers.stream()
-                    .filter(p -> p.getPeerData().getAddress().equals(peerAddress))
-                    .findFirst()
-                    .orElse(null);
+            Peer connectedPeer = null;
+            synchronized (this.connectedPeers) {
+                connectedPeer = this.connectedPeers.stream()
+                        .filter(p -> p.getPeerData().getAddress().equals(peerAddress))
+                        .findFirst()
+                        .orElse(null);
+            }
             boolean isConnected = (connectedPeer != null);
 
             boolean isHandshaked = this.getHandshakedPeers().stream()
@@ -1181,6 +1195,11 @@ public class Network {
         //ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
     }
 
+    public String getOurExternalIpAddress() {
+        // FUTURE: replace port if UPnP is active, as it will be more accurate
+        return this.ourExternalIpAddress;
+    }
+
 
     // Peer-management calls
 

src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java

@@ -1,6 +1,8 @@
 package org.qortal.network.message;
 
 import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.PeerData;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.Transformer;
 import org.qortal.utils.Serialization;
@@ -16,22 +18,38 @@ public class ArbitraryDataFileListMessage extends Message {
 
 	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
 	private static final int HASH_LENGTH = Transformer.SHA256_LENGTH;
+	private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE;
 
 	private final byte[] signature;
 	private final List<byte[]> hashes;
+	private Long requestTime;
+	private Integer requestHops;
+	private String peerAddress;
+	private Boolean isRelayPossible;
 
-	public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes) {
+
+	public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes, Long requestTime,
+										Integer requestHops, String peerAddress, boolean isRelayPossible) {
 		super(MessageType.ARBITRARY_DATA_FILE_LIST);
 
 		this.signature = signature;
 		this.hashes = hashes;
+		this.requestTime = requestTime;
+		this.requestHops = requestHops;
+		this.peerAddress = peerAddress;
+		this.isRelayPossible = isRelayPossible;
 	}
 
-	public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes) {
+	public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes, Long requestTime,
+										Integer requestHops, String peerAddress, boolean isRelayPossible) {
 		super(id, MessageType.ARBITRARY_DATA_FILE_LIST);
 
 		this.signature = signature;
 		this.hashes = hashes;
+		this.requestTime = requestTime;
+		this.requestHops = requestHops;
+		this.peerAddress = peerAddress;
+		this.isRelayPossible = isRelayPossible;
 	}
 
 	public List<byte[]> getHashes() {
@@ -48,9 +66,6 @@ public class ArbitraryDataFileListMessage extends Message {
 
 		int count = bytes.getInt();
 
-		if (bytes.remaining() != count * HASH_LENGTH)
-			return null;
-
 		List<byte[]> hashes = new ArrayList<>();
 		for (int i = 0; i < count; ++i) {
 
@@ -59,7 +74,26 @@ public class ArbitraryDataFileListMessage extends Message {
 			hashes.add(hash);
 		}
 
-		return new ArbitraryDataFileListMessage(id, signature, hashes);
+		Long requestTime = null;
+		Integer requestHops = null;
+		String peerAddress = null;
+		boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible
+
+		// The remaining fields are optional
+
+		if (bytes.hasRemaining()) {
+
+			requestTime = bytes.getLong();
+
+			requestHops = bytes.getInt();
+
+			peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH);
+
+			isRelayPossible = bytes.getInt() > 0;
+
+		}
+
+		return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible);
 	}
 
 	@Override
@@ -75,6 +109,20 @@ public class ArbitraryDataFileListMessage extends Message {
 				bytes.write(hash);
 			}
 
+			if (this.requestTime == null) { // To maintain backwards support
+				return bytes.toByteArray();
+			}
+
+			// The remaining fields are optional
+
+			bytes.write(Longs.toByteArray(this.requestTime));
+
+			bytes.write(Ints.toByteArray(this.requestHops));
+
+			Serialization.serializeSizedStringV2(bytes, this.peerAddress);
+
+			bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0));
+
 			return bytes.toByteArray();
 		} catch (IOException e) {
 			return null;
@@ -82,9 +130,49 @@ public class ArbitraryDataFileListMessage extends Message {
 	}
 
 	public ArbitraryDataFileListMessage cloneWithNewId(int newId) {
-		ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes);
+		ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes,
+				this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible);
 		clone.setId(newId);
 		return clone;
 	}
 
+	public void removeOptionalStats() {
+		this.requestTime = null;
+		this.requestHops = null;
+		this.peerAddress = null;
+		this.isRelayPossible = null;
+	}
+
+	public Long getRequestTime() {
+		return this.requestTime;
+	}
+
+	public void setRequestTime(Long requestTime) {
+		this.requestTime = requestTime;
+	}
+
+	public Integer getRequestHops() {
+		return this.requestHops;
+	}
+
+	public void setRequestHops(Integer requestHops) {
+		this.requestHops = requestHops;
+	}
+
+	public String getPeerAddress() {
+		return this.peerAddress;
+	}
+
+	public void setPeerAddress(String peerAddress) {
+		this.peerAddress = peerAddress;
+	}
+
+	public Boolean isRelayPossible() {
+		return this.isRelayPossible;
+	}
+
+	public void setIsRelayPossible(Boolean isRelayPossible) {
+		this.isRelayPossible = isRelayPossible;
+	}
+
 }

src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java

@@ -1,6 +1,8 @@
 package org.qortal.network.message;
 
 import com.google.common.primitives.Ints;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.repository.DataException;
 import org.qortal.transform.Transformer;
@@ -12,6 +14,8 @@ import java.nio.ByteBuffer;
 
 public class ArbitraryDataFileMessage extends Message {
 
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class);
+
 	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
 
 	private final byte[] signature;
@@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message {
 			return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
 		}
 		catch (DataException e) {
+			LOGGER.info("Unable to process received file: {}", e.getMessage());
 			return null;
 		}
 	}

src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java

@@ -0,0 +1,117 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * For requesting online accounts info from remote peer, given our list of online accounts.
+ *
+ * Different format to V1:
+ * V1 is: number of entries, then timestamp + pubkey for each entry
+ * V2 is: groups of: number of entries, timestamp, then pubkey for each entry
+ *
+ * Also V2 only builds online accounts message once!
+ */
+public class GetOnlineAccountsV2Message extends Message {
+	private List<OnlineAccountData> onlineAccounts;
+	private byte[] cachedData;
+
+	public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
+		this(-1, onlineAccounts);
+	}
+
+	private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
+		super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
+
+		this.onlineAccounts = onlineAccounts;
+	}
+
+	public List<OnlineAccountData> getOnlineAccounts() {
+		return this.onlineAccounts;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int accountCount = bytes.getInt();
+
+		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
+
+		while (accountCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < accountCount; ++i) {
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
+			}
+
+			if (bytes.hasRemaining()) {
+				accountCount = bytes.getInt();
+			} else {
+				// we've finished
+				accountCount = 0;
+			}
+		}
+
+		return new GetOnlineAccountsV2Message(id, onlineAccounts);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no online accounts
+		if (this.onlineAccounts.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+			OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+			Long timestamp = onlineAccountData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+					OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+
+					if (onlineAccountData.getTimestamp() == timestamp)
+						bytes.write(onlineAccountData.getPublicKey());
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/network/message/Message.java

@@ -78,6 +78,8 @@ public abstract class Message {
 
 		ONLINE_ACCOUNTS(80),
 		GET_ONLINE_ACCOUNTS(81),
+		ONLINE_ACCOUNTS_V2(82),
+		GET_ONLINE_ACCOUNTS_V2(83),
 
 		ARBITRARY_DATA(90),
 		GET_ARBITRARY_DATA(91),

src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java

@@ -0,0 +1,124 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * For sending online accounts info to remote peer.
+ *
+ * Different format to V1:
+ * V1 is: number of entries, then timestamp + sig + pubkey for each entry
+ * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
+ *
+ * Also V2 only builds online accounts message once!
+ */
+public class OnlineAccountsV2Message extends Message {
+	private List<OnlineAccountData> onlineAccounts;
+	private byte[] cachedData;
+
+	public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
+		this(-1, onlineAccounts);
+	}
+
+	private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
+		super(id, MessageType.ONLINE_ACCOUNTS_V2);
+
+		this.onlineAccounts = onlineAccounts;
+	}
+
+	public List<OnlineAccountData> getOnlineAccounts() {
+		return this.onlineAccounts;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int accountCount = bytes.getInt();
+
+		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
+
+		while (accountCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < accountCount; ++i) {
+				byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
+				bytes.get(signature);
+
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
+			}
+
+			if (bytes.hasRemaining()) {
+				accountCount = bytes.getInt();
+			} else {
+				// we've finished
+				accountCount = 0;
+			}
+		}
+
+		return new OnlineAccountsV2Message(id, onlineAccounts);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no online accounts
+		if (this.onlineAccounts.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+			OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+			Long timestamp = onlineAccountData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+					OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+
+					if (onlineAccountData.getTimestamp() == timestamp) {
+						bytes.write(onlineAccountData.getSignature());
+
+						bytes.write(onlineAccountData.getPublicKey());
+					}
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/settings/Settings.java

@@ -181,6 +181,8 @@ public class Settings {
 	private boolean isTestNet = false;
 	/** Port number for inbound peer-to-peer connections. */
 	private Integer listenPort;
+	/** Whether to attempt to open the listen port via UPnP */
+	private boolean uPnPEnabled = true;
 	/** Minimum number of peers to allow block minting / synchronization. */
 	private int minBlockchainPeers = 5;
 	/** Target number of outbound connections to peers we should make. */
@@ -195,7 +197,7 @@ public class Settings {
 	private int maxRetries = 2;
 
 	/** Minimum peer version number required in order to sync with them */
-	private String minPeerVersion = "3.0.1";
+	private String minPeerVersion = "3.1.0";
 	/** Whether to allow connections with peers below minPeerVersion
 	 * If true, we won't sync with them but they can still sync with us, and will show in the peers list
 	 * If false, sync will be blocked both ways, and they will not appear in the peers list */
@@ -629,6 +631,10 @@ public class Settings {
 		return this.bindAddress;
 	}
 
+	public boolean isUPnPEnabled() {
+		return this.uPnPEnabled;
+	}
+
 	public int getMinBlockchainPeers() {
 		return this.minBlockchainPeers;
 	}

src/main/java/org/qortal/transaction/TransferPrivsTransaction.java

@@ -58,7 +58,9 @@ public class TransferPrivsTransaction extends Transaction {
 			return ValidationResult.INVALID_ADDRESS;
 
 		// Check recipient is new account
-		if (this.repository.getAccountRepository().accountExists(this.transferPrivsTransactionData.getRecipient()))
+		AccountData recipientAccountData = this.repository.getAccountRepository().getAccount(this.transferPrivsTransactionData.getRecipient());
+		// Non-existent account data is OK, but if account data exists then reference needs to be null
+		if (recipientAccountData != null && recipientAccountData.getReference() != null)
 			return ValidationResult.ACCOUNT_ALREADY_EXISTS;
 
 		// Check sender has funds for fee

src/test/java/org/qortal/test/network/OnlineAccountsTests.java

@@ -0,0 +1,114 @@
+package org.qortal.test.network;
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.junit.Test;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.network.message.*;
+import org.qortal.transform.Transformer;
+
+import java.nio.ByteBuffer;
+import java.security.Security;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class OnlineAccountsTests {
+
+    private static final Random RANDOM = new Random();
+    static {
+        // This must go before any calls to LogManager/Logger
+        System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
+
+        Security.insertProviderAt(new BouncyCastleProvider(), 0);
+        Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
+    }
+
+
+    @Test
+    public void testGetOnlineAccountsV2() throws Message.MessageException {
+        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
+
+        Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
+
+        byte[] messageBytes = messageOut.toBytes();
+        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
+
+        GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
+
+        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
+
+        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
+        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
+
+        Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
+        byte[] oldMessageBytes = oldMessageOut.toBytes();
+
+        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
+
+        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
+                onlineAccountsOut.size(),
+                numTimestamps,
+                numTimestamps != 1 ? "s" : "",
+                oldMessageBytes.length,
+                messageBytes.length));
+    }
+
+    @Test
+    public void testOnlineAccountsV2() throws Message.MessageException {
+        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
+
+        Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
+
+        byte[] messageBytes = messageOut.toBytes();
+        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
+
+        OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
+
+        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
+
+        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
+        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
+
+        Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
+        byte[] oldMessageBytes = oldMessageOut.toBytes();
+
+        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
+
+        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
+                onlineAccountsOut.size(),
+                numTimestamps,
+                numTimestamps != 1 ? "s" : "",
+                oldMessageBytes.length,
+                messageBytes.length));
+    }
+
+    private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
+        List<OnlineAccountData> onlineAccounts = new ArrayList<>();
+
+        int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
+
+        for (int t = 0; t < numTimestamps; ++t) {
+            int numAccounts = RANDOM.nextInt(3000);
+
+            for (int a = 0; a < numAccounts; ++a) {
+                byte[] sig = null;
+                if (withSignatures) {
+                    sig = new byte[Transformer.SIGNATURE_LENGTH];
+                    RANDOM.nextBytes(sig);
+                }
+
+                byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+                RANDOM.nextBytes(pubkey);
+
+                onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
+            }
+        }
+
+        return onlineAccounts;
+    }
+
+}