Disabled auto fetching of metadata. To be re-enabled at a later date.

# Conflicts:
#	src/main/java/org/qortal/controller/Controller.java
#	src/main/java/org/qortal/network/message/Message.java

TestNets.md

@@ -41,13 +41,39 @@
 - Start up at least as many nodes as `minBlockchainPeers` (or adjust this value instead)
 - Probably best to perform API call `DELETE /peers/known`
 - Add other nodes via API call `POST /peers <peer-hostname-or-IP>`
-- Add minting private key to node(s) via API call `POST /admin/mintingaccounts <minting-private-key>`
-	This key must have corresponding `REWARD_SHARE` transaction in testnet genesis block
+- Add minting private key to nodes via API call `POST /admin/mintingaccounts <minting-private-key>`
+    The keys must have corresponding `REWARD_SHARE` transactions in testnet genesis block
+- You must have at least 2 separate minting keys and two separate nodes. Assign one minting key to each node.
+- Alternatively, comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java to allow for a single node and key.
 - Wait for genesis block timestamp to pass
 - A node should mint block 2 approximately 60 seconds after genesis block timestamp
 - Other testnet nodes will sync *as long as there is at least `minBlockchainPeers` peers with an "up-to-date" chain`
 - You can also use API call `POST /admin/forcesync <connected-peer-IP-and-port>` on stuck nodes
 
+## Single-node testnet
+
+A single-node testnet is possible with code modifications, for basic testing, or to more easily start a new testnet.
+To do so, follow these steps:
+- Comment out the `if (mintedLastBlock) { }` conditional in BlockMinter.java
+- Comment out the `minBlockchainPeers` validation in Settings.validate()
+- Set `minBlockchainPeers` to 0 in settings.json
+- Set `Synchronizer.RECOVERY_MODE_TIMEOUT` to `0`
+- All other steps should remain the same. Only a single reward share key is needed.
+- Remember to put these values back after introducing other nodes
+
+## Fixed network
+
+To restrict a testnet to a set of private nodes, you can use the "fixed network" feature.
+This ensures that the testnet nodes only communicate with each other and not other known peers.
+To do this, add the following setting to each testnet node, substituting the IP addresses:
+```
+"fixedNetwork": [
+  "192.168.0.101:62392",
+  "192.168.0.102:62392",
+  "192.168.0.103:62392"
+]
+```
+
 ## Dealing with stuck chain
 
 Maybe your nodes have been offline and no-one has minted a recent testnet block.

WindowsInstaller/Qortal.aip

@@ -17,10 +17,10 @@
     <ROW Property="Manufacturer" Value="Qortal"/>
     <ROW Property="MsiLogging" MultiBuildValue="DefaultBuild:vp"/>
     <ROW Property="NTP_GOOD" Value="false"/>
-    <ROW Property="ProductCode" Value="1033:{9BDE0BDF-72A2-44DA-8B55-E7C129DBE603} 1049:{F4FCC1D9-D286-4B3D-A50F-82034010A30F} 2052:{DBE9D682-F666-49BA-8B63-28C0AE06CBCA} 2057:{949F4DFE-E55C-4493-AAB6-5DB13E68C754} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{5FC8DCC3-BF9C-4D72-8C6D-940340ACD1B8} 1049:{1DEF14AB-2397-4517-B3C8-13221B921753} 2052:{B9E3C1DF-C92D-440A-9A21-869582F8585F} 2057:{91D69E7B-CA7D-4449-8E8A-F22DCEA546FC} " Type="16"/>
     <ROW Property="ProductLanguage" Value="2057"/>
     <ROW Property="ProductName" Value="Qortal"/>
-    <ROW Property="ProductVersion" Value="3.0.4" Type="32"/>
+    <ROW Property="ProductVersion" Value="3.1.1" Type="32"/>
     <ROW Property="RECONFIG_NTP" Value="true"/>
     <ROW Property="REMOVE_BLOCKCHAIN" Value="YES" Type="4"/>
     <ROW Property="REPAIR_BLOCKCHAIN" Value="YES" Type="4"/>
@@ -212,7 +212,7 @@
     <ROW Component="ADDITIONAL_LICENSE_INFO_71" ComponentId="{12A3ADBE-BB7A-496C-8869-410681E6232F}" Directory_="jdk.zipfs_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_71" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_8" ComponentId="{D53AD95E-CF96-4999-80FC-5812277A7456}" Directory_="java.naming_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_8" Type="0"/>
     <ROW Component="ADDITIONAL_LICENSE_INFO_9" ComponentId="{6B7EA9B0-5D17-47A8-B78C-FACE86D15E01}" Directory_="java.net.http_Dir" Attributes="0" KeyPath="ADDITIONAL_LICENSE_INFO_9" Type="0"/>
-    <ROW Component="AI_CustomARPName" ComponentId="{9B71A82D-8C25-40FD-806D-44BAD0B45AA2}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{42F5EC19-E46F-4299-B9F7-6E1112F6E4FB}" Directory_="APPDIR" Attributes="260" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{3644948D-AE0B-41BB-9FAF-A79E70490A08}" Directory_="APPDIR" Attributes="260" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{680DFDDE-3FB4-47A5-8FF5-934F576C6F91}" Directory_="APPDIR" Attributes="0"/>
     <ROW Component="AccessBridgeCallbacks.h" ComponentId="{288055D1-1062-47A3-AA44-5601B4E38AED}" Directory_="bridge_Dir" Attributes="0" KeyPath="AccessBridgeCallbacks.h" Type="0"/>

lib/com/dosse/WaifUPnP/1.1/WaifUPnP-1.1.pom

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.dosse</groupId>
+  <artifactId>WaifUPnP</artifactId>
+  <version>1.1</version>
+  <description>POM was created from install:install-file</description>
+</project>

lib/com/dosse/WaifUPnP/maven-metadata-local.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<metadata>
+  <groupId>com.dosse</groupId>
+  <artifactId>WaifUPnP</artifactId>
+  <versioning>
+    <release>1.1</release>
+    <versions>
+      <version>1.1</version>
+    </versions>
+    <lastUpdated>20220218200127</lastUpdated>
+  </versioning>
+</metadata>

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>3.0.4</version>
+	<version>3.1.1</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>
@@ -21,6 +21,7 @@
 		<dagger.version>1.2.2</dagger.version>
 		<guava.version>28.1-jre</guava.version>
 		<hsqldb.version>2.5.1</hsqldb.version>
+		<upnp.version>1.1</upnp.version>
 		<jersey.version>2.29.1</jersey.version>
 		<jetty.version>9.4.29.v20200521</jetty.version>
 		<log4j.version>2.17.1</log4j.version>
@@ -427,6 +428,12 @@
 			<artifactId>AT</artifactId>
 			<version>${ciyam-at.version}</version>
 		</dependency>
+		<!-- UPnP support -->
+		<dependency>
+			<groupId>com.dosse</groupId>
+			<artifactId>WaifUPnP</artifactId>
+			<version>${upnp.version}</version>
+		</dependency>
 		<!-- Bitcoin support -->
 		<dependency>
 			<groupId>org.bitcoinj</groupId>

src/main/java/org/qortal/account/Account.java

@@ -272,7 +272,7 @@ public class Account {
 	/**
 	 * Returns 'effective' minting level, or zero if reward-share does not exist.
 	 * <p>
-	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
+	 * this is being used on src/main/java/org/qortal/api/resource/AddressesResource.java to fulfil the online accounts api call
 	 * 
 	 * @param repository
 	 * @param rewardSharePublicKey
@@ -288,5 +288,26 @@ public class Account {
 		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
 		return rewardShareMinter.getEffectiveMintingLevel();
 	}
+	/**
+	 * Returns 'effective' minting level, with a fix for the zero level.
+	 * <p>
+	 * For founder accounts, this returns "founderEffectiveMintingLevel" from blockchain config.
+	 *
+	 * @param repository
+	 * @param rewardSharePublicKey
+	 * @return 0+
+	 * @throws DataException
+	 */
+	public static int getRewardShareEffectiveMintingLevelIncludingLevelZero(Repository repository, byte[] rewardSharePublicKey) throws DataException {
+		// Find actual minter and get their effective minting level
+		RewardShareData rewardShareData = repository.getAccountRepository().getRewardShare(rewardSharePublicKey);
+		if (rewardShareData == null)
+			return 0;
 
+		else if(!rewardShareData.getMinter().equals(rewardShareData.getRecipient()))//the minter is different than the recipient this means sponsorship
+			return 0;
+
+		Account rewardShareMinter = new Account(repository, rewardShareData.getMinter());
+		return rewardShareMinter.getEffectiveMintingLevel();
+	}
 }

src/main/java/org/qortal/api/ApiService.java

@@ -40,13 +40,7 @@ import org.glassfish.jersey.server.ResourceConfig;
 import org.glassfish.jersey.servlet.ServletContainer;
 import org.qortal.api.resource.AnnotationPostProcessor;
 import org.qortal.api.resource.ApiDefinition;
-import org.qortal.api.websocket.ActiveChatsWebSocket;
-import org.qortal.api.websocket.AdminStatusWebSocket;
-import org.qortal.api.websocket.BlocksWebSocket;
-import org.qortal.api.websocket.ChatMessagesWebSocket;
-import org.qortal.api.websocket.PresenceWebSocket;
-import org.qortal.api.websocket.TradeBotWebSocket;
-import org.qortal.api.websocket.TradeOffersWebSocket;
+import org.qortal.api.websocket.*;
 import org.qortal.settings.Settings;
 
 public class ApiService {
@@ -212,6 +206,9 @@ public class ApiService {
 			context.addServlet(ChatMessagesWebSocket.class, "/websockets/chat/messages");
 			context.addServlet(TradeOffersWebSocket.class, "/websockets/crosschain/tradeoffers");
 			context.addServlet(TradeBotWebSocket.class, "/websockets/crosschain/tradebot");
+			context.addServlet(TradePresenceWebSocket.class, "/websockets/crosschain/tradepresence");
+
+			// Deprecated
 			context.addServlet(PresenceWebSocket.class, "/websockets/presence");
 
 			// Start server

src/main/java/org/qortal/api/HTMLParser.java

@@ -28,6 +28,11 @@ public class HTMLParser {
             // Add base href tag
             String baseElement = String.format("<base href=\"%s\">", baseUrl);
             head.get(0).prepend(baseElement);
+
+            // Add meta charset tag
+            String metaCharsetElement = "<meta charset=\"UTF-8\">";
+            head.get(0).prepend(metaCharsetElement);
+
         }
         String html = document.html();
         this.data = html.getBytes();

src/main/java/org/qortal/api/domainmap/resource/DomainMapResource.java

@@ -17,7 +17,7 @@ import java.util.Map;
 
 
 @Path("/")
-@Tag(name = "Gateway")
+@Tag(name = "Domain Map")
 public class DomainMapResource {
 
     @Context HttpServletRequest request;

src/main/java/org/qortal/api/resource/AddressesResource.java

@@ -198,7 +198,7 @@ public class AddressesResource {
 
 			for (OnlineAccountData onlineAccountData : onlineAccounts) {
 				try {
-					final int minterLevel = Account.getRewardShareEffectiveMintingLevel(repository, onlineAccountData.getPublicKey());
+					final int minterLevel = Account.getRewardShareEffectiveMintingLevelIncludingLevelZero(repository, onlineAccountData.getPublicKey());
 
 					OnlineAccountLevel onlineAccountLevel = onlineAccountLevels.stream()
 							.filter(a -> a.getLevel() == minterLevel)

src/main/java/org/qortal/api/resource/AdminResource.java

@@ -315,6 +315,7 @@ public class AdminResource {
 
 			repository.getAccountRepository().save(mintingAccountData);
 			repository.saveChanges();
+			repository.exportNodeLocalData();//after adding new minting account let's persist it to the backup  MintingAccounts.json 
 		} catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
 		} catch (DataException e) {
@@ -355,6 +356,7 @@ public class AdminResource {
 				return "false";
 
 			repository.saveChanges();
+			repository.exportNodeLocalData();//after removing new minting account let's persist it to the backup  MintingAccounts.json 
 		} catch (IllegalArgumentException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_PRIVATE_KEY, e);
 		} catch (DataException e) {
@@ -546,7 +548,7 @@ public class AdminResource {
 	@Path("/repository/data")
 	@Operation(
 		summary = "Export sensitive/node-local data from repository.",
-		description = "Exports data to .script files on local machine"
+		description = "Exports data to .json files on local machine"
 	)
 	@ApiErrors({ApiError.INVALID_DATA, ApiError.REPOSITORY_ISSUE})
 	@SecurityRequirement(name = "apiKey")

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -12,6 +12,7 @@ import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 
 import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.ArrayList;
@@ -33,13 +34,14 @@ import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
 import org.qortal.arbitrary.*;
 import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
 import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
+import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
 import org.qortal.data.account.AccountData;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+import org.qortal.data.arbitrary.*;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -88,7 +90,8 @@ public class ArbitraryResource {
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse,
-			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) {
+			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
+			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata) {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 
@@ -110,9 +113,12 @@ public class ArbitraryResource {
 				return new ArrayList<>();
 			}
 
-			if (includeStatus != null && includeStatus == true) {
+			if (includeStatus != null && includeStatus) {
 				resources = this.addStatusToResources(resources);
 			}
+			if (includeMetadata != null && includeMetadata) {
+				resources = this.addMetadataToResources(resources);
+			}
 
 			return resources;
 
@@ -140,7 +146,8 @@ public class ArbitraryResource {
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse,
-			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) {
+			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
+			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata) {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 
@@ -153,9 +160,12 @@ public class ArbitraryResource {
 				return new ArrayList<>();
 			}
 
-			if (includeStatus != null && includeStatus == true) {
+			if (includeStatus != null && includeStatus) {
 				resources = this.addStatusToResources(resources);
 			}
+			if (includeMetadata != null && includeMetadata) {
+				resources = this.addMetadataToResources(resources);
+			}
 
 			return resources;
 
@@ -182,7 +192,8 @@ public class ArbitraryResource {
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse,
-			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus) {
+			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
+			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata) {
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 
@@ -206,9 +217,13 @@ public class ArbitraryResource {
 					List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
 							.getArbitraryResources(service, identifier, name, defaultRes, null, null, reverse);
 
-					if (includeStatus != null && includeStatus == true) {
+					if (includeStatus != null && includeStatus) {
 						resources = this.addStatusToResources(resources);
 					}
+					if (includeMetadata != null && includeMetadata) {
+						resources = this.addMetadataToResources(resources);
+					}
+
 					creatorName.resources = resources;
 				}
 			}
@@ -390,6 +405,28 @@ public class ArbitraryResource {
 		return Settings.getInstance().isRelayModeEnabled();
 	}
 
+	@GET
+	@Path("/categories")
+	@Operation(
+			summary = "List arbitrary transaction categories",
+			responses = {
+					@ApiResponse(
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryCategoryInfo.class))
+					)
+			}
+	)
+	@ApiErrors({ApiError.REPOSITORY_ISSUE})
+	public List<ArbitraryCategoryInfo> getCategories() {
+		List<ArbitraryCategoryInfo> categories = new ArrayList<>();
+		for (Category category : Category.values()) {
+			ArbitraryCategoryInfo arbitraryCategory = new ArbitraryCategoryInfo();
+			arbitraryCategory.id = category.toString();
+			arbitraryCategory.name = category.getName();
+			categories.add(arbitraryCategory);
+		}
+		return categories;
+	}
+
 	@GET
 	@Path("/hosted/transactions")
 	@Operation(
@@ -431,15 +468,24 @@ public class ArbitraryResource {
 	public List<ArbitraryResourceInfo> getHostedResources(
 			@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
+			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
-			@Parameter(ref = "offset") @QueryParam("offset") Integer offset) {
+			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
+			@QueryParam("query") String query) {
 		Security.checkApiCallAllowed(request);
 
 		List<ArbitraryResourceInfo> resources = new ArrayList<>();
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			
+			List<ArbitraryTransactionData> transactionDataList;
+
+			if (query == null || query.equals("")) {
+				transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset);
+			} else {
+				transactionDataList = ArbitraryDataStorageManager.getInstance().searchHostedTransactions(repository,query, limit, offset);
+			}
 
-			List<ArbitraryTransactionData> transactionDataList = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, limit, offset);
 			for (ArbitraryTransactionData transactionData : transactionDataList) {
 				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
 				arbitraryResourceInfo.name = transactionData.getName();
@@ -450,9 +496,12 @@ public class ArbitraryResource {
 				}
 			}
 
-			if (includeStatus != null && includeStatus == true) {
+			if (includeStatus != null && includeStatus) {
 				resources = this.addStatusToResources(resources);
 			}
+			if (includeMetadata != null && includeMetadata) {
+				resources = this.addMetadataToResources(resources);
+			}
 
 			return resources;
 
@@ -461,6 +510,8 @@ public class ArbitraryResource {
 		}
 	}
 
+
+
 	@DELETE
 	@Path("/resource/{service}/{name}/{identifier}")
 	@Operation(
@@ -624,6 +675,54 @@ public class ArbitraryResource {
 	}
 
 
+	// Metadata
+
+	@GET
+	@Path("/metadata/{service}/{name}/{identifier}")
+	@Operation(
+			summary = "Fetch raw metadata from resource with supplied service, name, identifier, and relative path",
+			responses = {
+					@ApiResponse(
+							description = "Path to file structure containing requested data",
+							content = @Content(
+									mediaType = MediaType.APPLICATION_JSON,
+									schema = @Schema(
+											implementation = ArbitraryDataTransactionMetadata.class
+									)
+							)
+					)
+			}
+	)
+	@SecurityRequirement(name = "apiKey")
+	public ArbitraryResourceMetadata getMetadata(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
+							  @PathParam("service") Service service,
+							  @PathParam("name") String name,
+							  @PathParam("identifier") String identifier) {
+		Security.checkApiCallAllowed(request);
+
+		ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
+
+		try {
+			ArbitraryDataTransactionMetadata transactionMetadata = ArbitraryMetadataManager.getInstance().fetchMetadata(resource, false);
+			if (transactionMetadata != null) {
+				ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata);
+				if (resourceMetadata != null) {
+					return resourceMetadata;
+				}
+				else {
+					// The metadata file doesn't contain title, description, category, or tags
+					throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.FILE_NOT_FOUND);
+				}
+			}
+		} catch (IllegalArgumentException e) {
+			// No metadata exists for this resource
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.FILE_NOT_FOUND, e.getMessage());
+		}
+
+		throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
+	}
+
+
 
 	// Upload data at supplied path
 
@@ -656,6 +755,10 @@ public class ArbitraryResource {
 	public String post(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 					   @PathParam("service") String serviceString,
 					   @PathParam("name") String name,
+					   @QueryParam("title") String title,
+					   @QueryParam("description") String description,
+					   @QueryParam("tags") List<String> tags,
+					   @QueryParam("category") Category category,
 					   String path) {
 		Security.checkApiCallAllowed(request);
 
@@ -663,7 +766,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false);
+		return this.upload(Service.valueOf(serviceString), name, null, path, null, null, false,
+				title, description, tags, category);
 	}
 
 	@POST
@@ -696,6 +800,10 @@ public class ArbitraryResource {
 					   @PathParam("service") String serviceString,
 					   @PathParam("name") String name,
 					   @PathParam("identifier") String identifier,
+					   @QueryParam("title") String title,
+					   @QueryParam("description") String description,
+					   @QueryParam("tags") List<String> tags,
+					   @QueryParam("category") Category category,
 					   String path) {
 		Security.checkApiCallAllowed(request);
 
@@ -703,7 +811,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Path not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false);
+		return this.upload(Service.valueOf(serviceString), name, identifier, path, null, null, false,
+				title, description, tags, category);
 	}
 
 
@@ -737,6 +846,10 @@ public class ArbitraryResource {
 	public String postBase64EncodedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 										@PathParam("service") String serviceString,
 										@PathParam("name") String name,
+										@QueryParam("title") String title,
+										@QueryParam("description") String description,
+										@QueryParam("tags") List<String> tags,
+										@QueryParam("category") Category category,
 										String base64) {
 		Security.checkApiCallAllowed(request);
 
@@ -744,7 +857,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false);
+		return this.upload(Service.valueOf(serviceString), name, null, null, null, base64, false,
+				title, description, tags, category);
 	}
 
 	@POST
@@ -775,6 +889,10 @@ public class ArbitraryResource {
 										@PathParam("service") String serviceString,
 										@PathParam("name") String name,
 										@PathParam("identifier") String identifier,
+										@QueryParam("title") String title,
+										@QueryParam("description") String description,
+										@QueryParam("tags") List<String> tags,
+										@QueryParam("category") Category category,
 										String base64) {
 		Security.checkApiCallAllowed(request);
 
@@ -782,7 +900,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false);
+		return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64, false,
+				title, description, tags, category);
 	}
 
 
@@ -815,6 +934,10 @@ public class ArbitraryResource {
 	public String postZippedData(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 								 @PathParam("service") String serviceString,
 								 @PathParam("name") String name,
+								 @QueryParam("title") String title,
+								 @QueryParam("description") String description,
+								 @QueryParam("tags") List<String> tags,
+								 @QueryParam("category") Category category,
 								 String base64Zip) {
 		Security.checkApiCallAllowed(request);
 
@@ -822,7 +945,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true);
+		return this.upload(Service.valueOf(serviceString), name, null, null, null, base64Zip, true,
+				title, description, tags, category);
 	}
 
 	@POST
@@ -853,6 +977,10 @@ public class ArbitraryResource {
 								 @PathParam("service") String serviceString,
 								 @PathParam("name") String name,
 								 @PathParam("identifier") String identifier,
+								 @QueryParam("title") String title,
+								 @QueryParam("description") String description,
+								 @QueryParam("tags") List<String> tags,
+								 @QueryParam("category") Category category,
 								 String base64Zip) {
 		Security.checkApiCallAllowed(request);
 
@@ -860,7 +988,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true);
+		return this.upload(Service.valueOf(serviceString), name, identifier, null, null, base64Zip, true,
+				title, description, tags, category);
 	}
 
 
@@ -896,6 +1025,10 @@ public class ArbitraryResource {
 	public String postString(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 							 @PathParam("service") String serviceString,
 							 @PathParam("name") String name,
+							 @QueryParam("title") String title,
+							 @QueryParam("description") String description,
+							 @QueryParam("tags") List<String> tags,
+							 @QueryParam("category") Category category,
 							 String string) {
 		Security.checkApiCallAllowed(request);
 
@@ -903,7 +1036,8 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false);
+		return this.upload(Service.valueOf(serviceString), name, null, null, string, null, false,
+				title, description, tags, category);
 	}
 
 	@POST
@@ -936,6 +1070,10 @@ public class ArbitraryResource {
 							 @PathParam("service") String serviceString,
 							 @PathParam("name") String name,
 							 @PathParam("identifier") String identifier,
+							 @QueryParam("title") String title,
+							 @QueryParam("description") String description,
+							 @QueryParam("tags") List<String> tags,
+							 @QueryParam("category") Category category,
 							 String string) {
 		Security.checkApiCallAllowed(request);
 
@@ -943,13 +1081,16 @@ public class ArbitraryResource {
 			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.INVALID_CRITERIA, "Data string not supplied");
 		}
 
-		return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false);
+		return this.upload(Service.valueOf(serviceString), name, identifier, null, string, null, false,
+				title, description, tags, category);
 	}
 
 
 	// Shared methods
 
-	private String upload(Service service, String name, String identifier, String path, String string, String base64, boolean zipped) {
+	private String upload(Service service, String name, String identifier,
+						  String path, String string, String base64, boolean zipped,
+						  String title, String description, List<String> tags, Category category) {
 		// Fetch public key from registered name
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			NameData nameData = repository.getNameRepository().fromName(name);
@@ -1013,7 +1154,8 @@ public class ArbitraryResource {
 
 			try {
 				ArbitraryDataTransactionBuilder transactionBuilder = new ArbitraryDataTransactionBuilder(
-						repository, publicKey58, Paths.get(path), name, null, service, identifier
+						repository, publicKey58, Paths.get(path), name, null, service, identifier,
+						title, description, tags, category
 				);
 
 				transactionBuilder.build();
@@ -1044,7 +1186,7 @@ public class ArbitraryResource {
 			// Loop until we have data
 			if (async) {
 				// Asynchronous
-				arbitraryDataReader.loadAsynchronously(false);
+				arbitraryDataReader.loadAsynchronously(false, 1);
 			}
 			else {
 				// Synchronous
@@ -1135,4 +1277,20 @@ public class ArbitraryResource {
 		}
 		return updatedResources;
 	}
+
+	private List<ArbitraryResourceInfo> addMetadataToResources(List<ArbitraryResourceInfo> resources) {
+		// Add metadata fields to each resource if they exist
+		List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
+		for (ArbitraryResourceInfo resourceInfo : resources) {
+			ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME,
+					resourceInfo.service, resourceInfo.identifier);
+			ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata();
+			ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata);
+			if (resourceMetadata != null) {
+				resourceInfo.metadata = resourceMetadata;
+			}
+			updatedResources.add(resourceInfo);
+		}
+		return updatedResources;
+	}
 }

src/main/java/org/qortal/api/resource/CrossChainBitcoinResource.java

@@ -122,7 +122,7 @@ public class CrossChainBitcoinResource {
 	@Path("/send")
 	@Operation(
 		summary = "Sends BTC from hierarchical, deterministic BIP32 wallet to specific address",
-		description = "Currently only supports 'legacy' P2PKH Bitcoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
+		description = "Currently supports 'legacy' P2PKH Bitcoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(

src/main/java/org/qortal/api/resource/CrossChainLitecoinResource.java

@@ -122,7 +122,7 @@ public class CrossChainLitecoinResource {
 	@Path("/send")
 	@Operation(
 		summary = "Sends LTC from hierarchical, deterministic BIP32 wallet to specific address",
-		description = "Currently only supports 'legacy' P2PKH Litecoin addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
+		description = "Currently supports 'legacy' P2PKH Litecoin addresses and Native SegWit (P2WPKH) addresses. Supply BIP32 'm' private key in base58, starting with 'xprv' for mainnet, 'tprv' for testnet",
 		requestBody = @RequestBody(
 			required = true,
 			content = @Content(

src/main/java/org/qortal/api/resource/CrossChainResource.java

@@ -25,6 +25,7 @@ import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.Security;
 import org.qortal.api.model.CrossChainCancelRequest;
 import org.qortal.api.model.CrossChainTradeSummary;
+import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crosschain.SupportedBlockchain;
 import org.qortal.crosschain.ACCT;
 import org.qortal.crosschain.AcctMode;
@@ -120,6 +121,8 @@ public class CrossChainResource {
 				crossChainTrades = crossChainTrades.subList(0, upperLimit);
 			}
 
+			crossChainTrades.stream().forEach(CrossChainResource::decorateTradeDataWithPresence);
+
 			return crossChainTrades;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
@@ -151,7 +154,11 @@ public class CrossChainResource {
 			if (acct == null)
 				throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
 
-			return acct.populateTradeData(repository, atData);
+			CrossChainTradeData crossChainTradeData = acct.populateTradeData(repository, atData);
+
+			decorateTradeDataWithPresence(crossChainTradeData);
+
+			return crossChainTradeData;
 		} catch (DataException e) {
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
 		}
@@ -486,4 +493,7 @@ public class CrossChainResource {
 		}
 	}
 
+	private static void decorateTradeDataWithPresence(CrossChainTradeData crossChainTradeData) {
+		TradeBot.getInstance().decorateTradeDataWithPresence(crossChainTradeData);
+	}
 }

src/main/java/org/qortal/api/resource/PeersResource.java

@@ -354,7 +354,7 @@ public class PeersResource {
 
 		List<Peer> connectedPeers = Network.getInstance().getConnectedPeers().stream().collect(Collectors.toList());
 		for (Peer peer : connectedPeers) {
-			if (peer.isOutbound()) {
+			if (!peer.isOutbound()) {
 				peersSummary.inboundConnections++;
 			}
 			else {

src/main/java/org/qortal/api/resource/RenderResource.java

@@ -74,7 +74,9 @@ public class RenderResource {
         Method method = Method.PUT;
         Compression compression = Compression.ZIP;
 
-        ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), null, Service.WEBSITE, null, method, compression);
+        ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath),
+                null, Service.WEBSITE, null, method, compression,
+                null, null, null, null);
         try {
             arbitraryDataWriter.save();
         } catch (IOException | DataException | InterruptedException | MissingDataException e) {

src/main/java/org/qortal/api/resource/TransactionsResource.java

@@ -638,7 +638,10 @@ public class TransactionsResource {
 		ApiError.BLOCKCHAIN_NEEDS_SYNC, ApiError.INVALID_SIGNATURE, ApiError.INVALID_DATA, ApiError.TRANSFORMATION_ERROR, ApiError.REPOSITORY_ISSUE
 	})
 	public String processTransaction(String rawBytes58) {
-		if (!Controller.getInstance().isUpToDate())
+		// Only allow a transaction to be processed if our latest block is less than 30 minutes old
+		// If older than this, we should first wait until the blockchain is synced
+		final Long minLatestBlockTimestamp = NTP.getTime() - (30 * 60 * 1000L);
+		if (!Controller.getInstance().isUpToDate(minLatestBlockTimestamp))
 			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.BLOCKCHAIN_NEEDS_SYNC);
 
 		byte[] rawBytes = Base58.decode(rawBytes58);

src/main/java/org/qortal/api/websocket/PresenceWebSocket.java

@@ -20,6 +20,7 @@ import org.eclipse.jetty.websocket.api.annotations.OnWebSocketMessage;
 import org.eclipse.jetty.websocket.api.annotations.WebSocket;
 import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.transaction.PresenceTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -99,13 +100,13 @@ public class PresenceWebSocket extends ApiWebSocket implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		// We use NewBlockEvent as a proxy for 1-minute timer
-		if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Controller.NewBlockEvent))
+		// We use Synchronizer.NewChainTipEvent as a proxy for 1-minute timer
+		if (!(event instanceof Controller.NewTransactionEvent) && !(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
 		removeOldEntries();
 
-		if (event instanceof Controller.NewBlockEvent)
+		if (event instanceof Synchronizer.NewChainTipEvent)
 			// We only wanted a chance to cull old entries
 			return;
 

src/main/java/org/qortal/api/websocket/TradeOffersWebSocket.java

@@ -23,6 +23,7 @@ import org.eclipse.jetty.websocket.api.annotations.WebSocket;
 import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
 import org.qortal.api.model.CrossChainOfferSummary;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.crosschain.SupportedBlockchain;
 import org.qortal.crosschain.ACCT;
 import org.qortal.crosschain.AcctMode;
@@ -80,10 +81,10 @@ public class TradeOffersWebSocket extends ApiWebSocket implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		if (!(event instanceof Controller.NewBlockEvent))
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
-		BlockData blockData = ((Controller.NewBlockEvent) event).getBlockData();
+		BlockData blockData = ((Synchronizer.NewChainTipEvent) event).getNewChainTip();
 
 		// Process any new info
 

src/main/java/org/qortal/api/websocket/TradePresenceWebSocket.java

@@ -0,0 +1,137 @@
+package org.qortal.api.websocket;
+
+import org.eclipse.jetty.websocket.api.Session;
+import org.eclipse.jetty.websocket.api.annotations.*;
+import org.eclipse.jetty.websocket.servlet.WebSocketServletFactory;
+import org.qortal.controller.Controller;
+import org.qortal.controller.tradebot.TradeBot;
+import org.qortal.data.network.TradePresenceData;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
+import org.qortal.event.Listener;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.*;
+
+@WebSocket
+@SuppressWarnings("serial")
+public class TradePresenceWebSocket extends ApiWebSocket implements Listener {
+
+	/** Map key is public key in base58, map value is trade presence */
+	private static final Map<String, TradePresenceData> currentEntries = Collections.synchronizedMap(new HashMap<>());
+
+	@Override
+	public void configure(WebSocketServletFactory factory) {
+		factory.register(TradePresenceWebSocket.class);
+
+		populateCurrentInfo();
+
+		EventBus.INSTANCE.addListener(this::listen);
+	}
+
+	@Override
+	public void listen(Event event) {
+		// XXX - Suggest we change this to something like Synchronizer.NewChainTipEvent?
+		// We use NewBlockEvent as a proxy for 1-minute timer
+		if (!(event instanceof TradeBot.TradePresenceEvent) && !(event instanceof Controller.NewBlockEvent))
+			return;
+
+		removeOldEntries();
+
+		if (event instanceof Controller.NewBlockEvent)
+			// We only wanted a chance to cull old entries
+			return;
+
+		TradePresenceData tradePresence = ((TradeBot.TradePresenceEvent) event).getTradePresenceData();
+
+		boolean somethingChanged = mergePresence(tradePresence);
+
+		if (!somethingChanged)
+			// nothing changed
+			return;
+
+		List<TradePresenceData> tradePresences = Collections.singletonList(tradePresence);
+
+		// Notify sessions
+		for (Session session : getSessions()) {
+			sendTradePresences(session, tradePresences);
+		}
+	}
+
+	@OnWebSocketConnect
+	@Override
+	public void onWebSocketConnect(Session session) {
+		Map<String, List<String>> queryParams = session.getUpgradeRequest().getParameterMap();
+
+		List<TradePresenceData> tradePresences;
+
+		synchronized (currentEntries) {
+			tradePresences = List.copyOf(currentEntries.values());
+		}
+
+		if (!sendTradePresences(session, tradePresences)) {
+			session.close(4002, "websocket issue");
+			return;
+		}
+
+		super.onWebSocketConnect(session);
+	}
+
+	@OnWebSocketClose
+	@Override
+	public void onWebSocketClose(Session session, int statusCode, String reason) {
+		// clean up
+		super.onWebSocketClose(session, statusCode, reason);
+	}
+
+	@OnWebSocketError
+	public void onWebSocketError(Session session, Throwable throwable) {
+		/* ignored */
+	}
+
+	@OnWebSocketMessage
+	public void onWebSocketMessage(Session session, String message) {
+		/* ignored */
+	}
+
+	private boolean sendTradePresences(Session session, List<TradePresenceData> tradePresences) {
+		try {
+			StringWriter stringWriter = new StringWriter();
+			marshall(stringWriter, tradePresences);
+
+			String output = stringWriter.toString();
+			session.getRemote().sendStringByFuture(output);
+		} catch (IOException e) {
+			// No output this time?
+			return false;
+		}
+
+		return true;
+	}
+
+	private static void populateCurrentInfo() {
+		// We want ALL trade presences
+		TradeBot.getInstance().getAllTradePresences().stream()
+				.forEach(TradePresenceWebSocket::mergePresence);
+	}
+
+	/** Merge trade presence into cache of current entries, returns true if cache was updated. */
+	private static boolean mergePresence(TradePresenceData tradePresence) {
+		// Put/replace for this publickey making sure we keep newest timestamp
+		String pubKey58 = Base58.encode(tradePresence.getPublicKey());
+
+		TradePresenceData newEntry = currentEntries.compute(pubKey58, (k, v) -> v == null || v.getTimestamp() < tradePresence.getTimestamp() ? tradePresence : v);
+
+		return newEntry == tradePresence;
+	}
+
+	private static void removeOldEntries() {
+		long now = NTP.getTime();
+
+		currentEntries.values().removeIf(v -> v.getTimestamp() < now);
+	}
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@@ -13,8 +13,11 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
     private final Long creationTimestamp;
     private Long buildStartTimestamp = null;
     private Long buildEndTimestamp = null;
+    private Integer priority = 0;
     private boolean failed = false;
 
+    private static int HIGH_PRIORITY_THRESHOLD = 5;
+
     /* The maximum amount of time to spend on a single build */
     // TODO: interrupt an in-progress build
     public static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
@@ -27,13 +30,20 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
         this.creationTimestamp = NTP.getTime();
     }
 
+    public void prepareForBuild() {
+        this.buildStartTimestamp = NTP.getTime();
+    }
+
     public void build() throws IOException, DataException, MissingDataException {
         Long now = NTP.getTime();
         if (now == null) {
+            this.buildStartTimestamp = null;
             throw new DataException("NTP time hasn't synced yet");
         }
 
-        this.buildStartTimestamp = now;
+        if (this.buildStartTimestamp == null) {
+            this.buildStartTimestamp = now;
+        }
         ArbitraryDataReader arbitraryDataReader =
                 new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service, this.identifier);
 
@@ -70,6 +80,21 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
         return this.buildStartTimestamp;
     }
 
+    public Integer getPriority() {
+        if (this.priority != null) {
+            return this.priority;
+        }
+        return 0;
+    }
+
+    public void setPriority(Integer priority) {
+        this.priority = priority;
+    }
+
+    public boolean isHighPriority() {
+        return this.priority >= HIGH_PRIORITY_THRESHOLD;
+    }
+
     public void setFailed(boolean failed) {
         this.failed = failed;
     }

src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java

@@ -61,6 +61,9 @@ public class ArbitraryDataCache {
         }
 
         // No need to invalidate the cache
+        // Remember that it's up to date, so that we won't check again for a while
+        ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
+
         return false;
     }
 
@@ -84,14 +87,7 @@ public class ArbitraryDataCache {
 
         // If the state's sig doesn't match the latest transaction's sig, we need to invalidate
         // This means that an updated layer is available
-        if (this.shouldInvalidateDueToSignatureMismatch()) {
-
-            // Add to the in-memory cache first, so that we won't check again for a while
-            ArbitraryDataManager.getInstance().addResourceToCache(this.getArbitraryDataResource());
-            return true;
-        }
-
-        return false;
+        return this.shouldInvalidateDueToSignatureMismatch();
     }
 
     /**

src/main/java/org/qortal/arbitrary/ArbitraryDataFile.java

@@ -366,6 +366,21 @@ public class ArbitraryDataFile {
         return false;
     }
 
+    public boolean delete(int attempts) {
+        // Keep trying to delete the data until it is deleted, or we reach 10 attempts
+        for (int i=0; i<attempts; i++) {
+            if (this.delete()) {
+                return true;
+            }
+            try {
+                Thread.sleep(1000L);
+            } catch (InterruptedException e) {
+                // Fall through to exit method
+            }
+        }
+        return false;
+    }
+
     public boolean deleteAllChunks() {
         boolean success = false;
 
@@ -771,6 +786,10 @@ public class ArbitraryDataFile {
         this.loadMetadata();
     }
 
+    public ArbitraryDataTransactionMetadata getMetadata() {
+        return this.metadata;
+    }
+
     @Override
     public String toString() {
         return this.shortHash58();

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -126,7 +126,7 @@ public class ArbitraryDataReader {
      * @param overwrite - set to true to force rebuild an existing cache
      * @return true if added or already present in queue; false if not
      */
-    public boolean loadAsynchronously(boolean overwrite) {
+    public boolean loadAsynchronously(boolean overwrite, int priority) {
         ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
                 this.resourceId, this.resourceIdType, this.service, this.identifier);
         if (cache.isCachedDataAvailable()) {
@@ -135,7 +135,9 @@ public class ArbitraryDataReader {
             return true;
         }
 
-        return ArbitraryDataBuildManager.getInstance().addToBuildQueue(this.createQueueItem());
+        ArbitraryDataBuildQueueItem item = this.createQueueItem();
+        item.setPriority(priority);
+        return ArbitraryDataBuildManager.getInstance().addToBuildQueue(item);
     }
 
     /**
@@ -468,12 +470,18 @@ public class ArbitraryDataReader {
             throw new DataException(String.format("Unable to unzip file: %s", e.getMessage()));
         }
 
-        // Replace filePath pointer with the uncompressed file path
+        if (!this.uncompressedPath.toFile().exists()) {
+            throw new DataException(String.format("Unable to unzip file: %s", this.filePath));
+        }
+
+        // Delete original compressed file
         if (FilesystemUtils.pathInsideDataOrTempPath(this.filePath)) {
             if (Files.exists(this.filePath)) {
                 Files.delete(this.filePath);
             }
         }
+
+        // Replace filePath pointer with the uncompressed file path
         this.filePath = this.uncompressedPath;
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -76,7 +76,7 @@ public class ArbitraryDataRenderer {
             if (!arbitraryDataReader.isCachedDataAvailable()) {
                 // If async is requested, show a loading screen whilst build is in progress
                 if (async) {
-                    arbitraryDataReader.loadAsynchronously(false);
+                    arbitraryDataReader.loadAsynchronously(false, 10);
                     return this.getLoadingResponse(service, resourceId);
                 }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -3,6 +3,7 @@ package org.qortal.arbitrary;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
@@ -37,6 +38,7 @@ public class ArbitraryDataResource {
 
     private List<ArbitraryTransactionData> transactions;
     private ArbitraryTransactionData latestPutTransaction;
+    private ArbitraryTransactionData latestTransaction;
     private int layerCount;
     private Integer localChunkCount = null;
     private Integer totalChunkCount = null;
@@ -105,6 +107,33 @@ public class ArbitraryDataResource {
         return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount);
     }
 
+    public ArbitraryDataTransactionMetadata getLatestTransactionMetadata() {
+        this.fetchLatestTransaction();
+
+        if (latestTransaction != null) {
+            byte[] signature = latestTransaction.getSignature();
+            byte[] metadataHash = latestTransaction.getMetadataHash();
+            if (metadataHash == null) {
+                // This resource doesn't have metadata
+                return null;
+            }
+
+            try {
+                ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                if (metadataFile.exists()) {
+                    ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+                    transactionMetadata.read();
+                    return transactionMetadata;
+                }
+
+            } catch (DataException | IOException e) {
+                // Do nothing
+            }
+        }
+
+        return null;
+    }
+
     public boolean delete() {
         try {
             this.fetchTransactions();
@@ -306,6 +335,32 @@ public class ArbitraryDataResource {
 
             this.transactions = transactionDataList;
             this.layerCount = transactionDataList.size();
+
+        } catch (DataException e) {
+            LOGGER.info(String.format("Repository error when fetching transactions for resource %s: %s", this, e.getMessage()));
+        }
+    }
+
+    private void fetchLatestTransaction() {
+        if (this.latestTransaction != null) {
+            // Already fetched
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Get the most recent transaction
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(this.resourceId, this.service, null, this.identifier);
+            if (latestTransaction == null) {
+                String message = String.format("Couldn't find transaction for name %s, service %s and identifier %s",
+                        this.resourceId, this.service, this.identifierString());
+                throw new DataException(message);
+            }
+            this.latestTransaction = latestTransaction;
+
+        } catch (DataException e) {
+            LOGGER.info(String.format("Repository error when fetching latest transaction for resource %s: %s", this, e.getMessage()));
         }
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -6,8 +6,9 @@ import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
 import org.qortal.arbitrary.ArbitraryDataDiff.*;
 import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.block.BlockChain;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.PaymentData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -27,6 +28,7 @@ import java.io.IOException;
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 import java.util.Random;
 
 public class ArbitraryDataTransactionBuilder {
@@ -51,13 +53,20 @@ public class ArbitraryDataTransactionBuilder {
     private final String identifier;
     private final Repository repository;
 
+    // Metadata
+    private final String title;
+    private final String description;
+    private final List<String> tags;
+    private final Category category;
+
     private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
 
     private ArbitraryTransactionData arbitraryTransactionData;
     private ArbitraryDataFile arbitraryDataFile;
 
     public ArbitraryDataTransactionBuilder(Repository repository, String publicKey58, Path path, String name,
-                                           Method method, Service service, String identifier) {
+                                           Method method, Service service, String identifier,
+                                           String title, String description, List<String> tags, Category category) {
         this.repository = repository;
         this.publicKey58 = publicKey58;
         this.path = path;
@@ -70,6 +79,12 @@ public class ArbitraryDataTransactionBuilder {
             identifier = null;
         }
         this.identifier = identifier;
+
+        // Metadata (optional)
+        this.title = ArbitraryDataTransactionMetadata.limitTitle(title);
+        this.description = ArbitraryDataTransactionMetadata.limitDescription(description);
+        this.tags = ArbitraryDataTransactionMetadata.limitTags(tags);
+        this.category = category;
     }
 
     public void build() throws DataException {
@@ -108,6 +123,10 @@ public class ArbitraryDataTransactionBuilder {
             return Method.PUT;
         }
 
+        // Get existing metadata and see if it matches the new metadata
+        ArbitraryDataResource resource = new ArbitraryDataResource(this.name, ResourceIdType.NAME, this.service, this.identifier);
+        ArbitraryDataTransactionMetadata existingMetadata = resource.getLatestTransactionMetadata();
+
         try {
             // Check layer count
             int layerCount = reader.getLayerCount();
@@ -118,7 +137,23 @@ public class ArbitraryDataTransactionBuilder {
 
             // Check size of differences between this layer and previous layer
             ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(reader.getFilePath(), this.path, reader.getLatestSignature());
-            patch.create();
+            try {
+                patch.create();
+            }
+            catch (DataException | IOException e) {
+                // Handle matching states separately, as it's best to block transactions with duplicate states
+                if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) {
+                    // Only throw an exception if the metadata is also identical, as well as the data
+                    if (this.isMetadataEqual(existingMetadata)) {
+                        throw new DataException(e.getMessage());
+                    }
+                }
+
+                LOGGER.info("Caught exception when creating patch: {}", e.getMessage());
+                LOGGER.info("Unable to load existing resource - using PUT to overwrite it.");
+                return Method.PUT;
+            }
+
             long diffSize = FilesystemUtils.getDirectorySize(patch.getFinalPath());
             long existingStateSize = FilesystemUtils.getDirectorySize(reader.getFilePath());
             double difference = (double) diffSize / (double) existingStateSize;
@@ -155,11 +190,8 @@ public class ArbitraryDataTransactionBuilder {
             // State is appropriate for a PATCH transaction
             return Method.PATCH;
         }
-        catch (IOException | DataException e) {
-            // Handle matching states separately, as it's best to block transactions with duplicate states
-            if (e.getMessage().equals("Current state matches previous state. Nothing to do.")) {
-                throw new DataException(e.getMessage());
-            }
+        catch (IOException e) {
+            // IMPORTANT: Don't catch DataException here, as they must be passed to the caller
             LOGGER.info("Caught exception: {}", e.getMessage());
             LOGGER.info("Unable to load existing resource - using PUT to overwrite it.");
             return Method.PUT;
@@ -200,7 +232,8 @@ public class ArbitraryDataTransactionBuilder {
             // FUTURE? Use zip compression for directories, or no compression for single files
             // Compression compression = (path.toFile().isDirectory()) ? Compression.ZIP : Compression.NONE;
 
-            ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method, compression);
+            ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, identifier, method,
+                    compression, title, description, tags, category);
             try {
                 arbitraryDataWriter.setChunkSize(this.chunkSize);
                 arbitraryDataWriter.save();
@@ -253,6 +286,22 @@ public class ArbitraryDataTransactionBuilder {
 
     }
 
+    private boolean isMetadataEqual(ArbitraryDataTransactionMetadata existingMetadata) {
+        if (!Objects.equals(existingMetadata.getTitle(), this.title)) {
+            return false;
+        }
+        if (!Objects.equals(existingMetadata.getDescription(), this.description)) {
+            return false;
+        }
+        if (!Objects.equals(existingMetadata.getCategory(), this.category)) {
+            return false;
+        }
+        if (!Objects.equals(existingMetadata.getTags(), this.tags)) {
+            return false;
+        }
+        return true;
+    }
+
     public void computeNonce() throws DataException {
         if (this.arbitraryTransactionData == null) {
             throw new DataException("Arbitrary transaction data is required to compute nonce");

src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java

@@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -28,6 +29,10 @@ import java.nio.file.Paths;
 import java.security.InvalidAlgorithmParameterException;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Objects;
 
 public class ArbitraryDataWriter {
 
@@ -40,6 +45,12 @@ public class ArbitraryDataWriter {
     private final Method method;
     private final Compression compression;
 
+    // Metadata
+    private final String title;
+    private final String description;
+    private final List<String> tags;
+    private final Category category;
+
     private int chunkSize = ArbitraryDataFile.CHUNK_SIZE;
 
     private SecretKey aesKey;
@@ -50,7 +61,8 @@ public class ArbitraryDataWriter {
     private Path compressedPath;
     private Path encryptedPath;
 
-    public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression) {
+    public ArbitraryDataWriter(Path filePath, String name, Service service, String identifier, Method method, Compression compression,
+                               String title, String description, List<String> tags, Category category) {
         this.filePath = filePath;
         this.name = name;
         this.service = service;
@@ -62,6 +74,12 @@ public class ArbitraryDataWriter {
             identifier = null;
         }
         this.identifier = identifier;
+
+        // Metadata (optional)
+        this.title = ArbitraryDataTransactionMetadata.limitTitle(title);
+        this.description = ArbitraryDataTransactionMetadata.limitDescription(description);
+        this.tags = ArbitraryDataTransactionMetadata.limitTags(tags);
+        this.category = category;
     }
 
     public void save() throws IOException, DataException, InterruptedException, MissingDataException {
@@ -258,12 +276,16 @@ public class ArbitraryDataWriter {
 
     private void createMetadataFile() throws IOException, DataException {
         // If we have at least one chunk, we need to create an index file containing their hashes
-        if (this.arbitraryDataFile.chunkCount() > 1) {
+        if (this.needsMetadataFile()) {
             // Create the JSON file
             Path chunkFilePath = Paths.get(this.workingPath.toString(), "metadata.json");
-            ArbitraryDataTransactionMetadata chunkMetadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
-            chunkMetadata.setChunks(this.arbitraryDataFile.chunkHashList());
-            chunkMetadata.write();
+            ArbitraryDataTransactionMetadata metadata = new ArbitraryDataTransactionMetadata(chunkFilePath);
+            metadata.setTitle(this.title);
+            metadata.setDescription(this.description);
+            metadata.setTags(this.tags);
+            metadata.setCategory(this.category);
+            metadata.setChunks(this.arbitraryDataFile.chunkHashList());
+            metadata.write();
 
             // Create an ArbitraryDataFile from the JSON file (we don't have a signature yet)
             ArbitraryDataFile metadataFile = ArbitraryDataFile.fromPath(chunkFilePath, null);
@@ -308,6 +330,20 @@ public class ArbitraryDataWriter {
                     throw new DataException(String.format("Missing chunk %s in metadata file", Base58.encode(chunk)));
                 }
             }
+
+            // Check that the metadata is correct
+            if (!Objects.equals(metadata.getTitle(), this.title)) {
+                throw new DataException("Metadata mismatch: title");
+            }
+            if (!Objects.equals(metadata.getDescription(), this.description)) {
+                throw new DataException("Metadata mismatch: description");
+            }
+            if (!Objects.equals(metadata.getTags(), this.tags)) {
+                throw new DataException("Metadata mismatch: tags");
+            }
+            if (!Objects.equals(metadata.getCategory(), this.category)) {
+                throw new DataException("Metadata mismatch: category");
+            }
         }
     }
 
@@ -330,6 +366,16 @@ public class ArbitraryDataWriter {
         }
     }
 
+    private boolean needsMetadataFile() {
+        if (this.arbitraryDataFile.chunkCount() > 1) {
+            return true;
+        }
+        if (this.title != null || this.description != null || this.tags != null || this.category != null) {
+            return true;
+        }
+        return false;
+    }
+
 
     public ArbitraryDataFile getArbitraryDataFile() {
         return this.arbitraryDataFile;

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataTransactionMetadata.java

@@ -2,17 +2,28 @@ package org.qortal.arbitrary.metadata;
 
 import org.json.JSONArray;
 import org.json.JSONObject;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.repository.DataException;
 import org.qortal.utils.Base58;
 
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Iterator;
 import java.util.List;
 
 public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
 
     private List<byte[]> chunks;
+    private String title;
+    private String description;
+    private List<String> tags;
+    private Category category;
+
+    private static int MAX_TITLE_LENGTH = 80;
+    private static int MAX_DESCRIPTION_LENGTH = 500;
+    private static int MAX_TAG_LENGTH = 20;
+    private static int MAX_TAGS_COUNT = 5;
 
     public ArbitraryDataTransactionMetadata(Path filePath) {
         super(filePath);
@@ -25,10 +36,37 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
             throw new DataException("Transaction metadata JSON string is null");
         }
 
+        JSONObject metadata = new JSONObject(this.jsonString);
+
+        if (metadata.has("title")) {
+            this.title = metadata.getString("title");
+        }
+
+        if (metadata.has("description")) {
+            this.description = metadata.getString("description");
+        }
+
+        List<String> tagsList = new ArrayList<>();
+        if (metadata.has("tags")) {
+            JSONArray tags = metadata.getJSONArray("tags");
+            if (tags != null) {
+                for (int i=0; i<tags.length(); i++) {
+                    String tag = tags.getString(i);
+                    if (tag != null) {
+                        tagsList.add(tag);
+                    }
+                }
+            }
+            this.tags = tagsList;
+        }
+
+        if (metadata.has("category")) {
+            this.category = Category.uncategorizedValueOf(metadata.getString("category"));
+        }
+
         List<byte[]> chunksList = new ArrayList<>();
-        JSONObject cache = new JSONObject(this.jsonString);
-        if (cache.has("chunks")) {
-            JSONArray chunks = cache.getJSONArray("chunks");
+        if (metadata.has("chunks")) {
+            JSONArray chunks = metadata.getJSONArray("chunks");
             if (chunks != null) {
                 for (int i=0; i<chunks.length(); i++) {
                     String chunk = chunks.getString(i);
@@ -45,6 +83,26 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
     protected void buildJson() {
         JSONObject outer = new JSONObject();
 
+        if (this.title != null && !this.title.isEmpty()) {
+            outer.put("title", this.title);
+        }
+
+        if (this.description != null && !this.description.isEmpty()) {
+            outer.put("description", this.description);
+        }
+
+        JSONArray tags = new JSONArray();
+        if (this.tags != null) {
+            for (String tag : this.tags) {
+                tags.put(tag);
+            }
+            outer.put("tags", tags);
+        }
+
+        if (this.category != null) {
+            outer.put("category", this.category.toString());
+        }
+
         JSONArray chunks = new JSONArray();
         if (this.chunks != null) {
             for (byte[] chunk : this.chunks) {
@@ -66,6 +124,38 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
         return this.chunks;
     }
 
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public String getTitle() {
+        return this.title;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setCategory(Category category) {
+        this.category = category;
+    }
+
+    public Category getCategory() {
+        return this.category;
+    }
+
     public boolean containsChunk(byte[] chunk) {
         for (byte[] c : this.chunks) {
             if (Arrays.equals(c, chunk)) {
@@ -75,4 +165,61 @@ public class ArbitraryDataTransactionMetadata extends ArbitraryDataMetadata {
         return false;
     }
 
+
+    // Static helper methods
+
+    public static String limitTitle(String title) {
+        if (title == null) {
+            return null;
+        }
+        if (title.isEmpty()) {
+            return null;
+        }
+
+        return title.substring(0, Math.min(title.length(), MAX_TITLE_LENGTH));
+    }
+
+    public static String limitDescription(String description) {
+        if (description == null) {
+            return null;
+        }
+        if (description.isEmpty()) {
+            return null;
+        }
+
+        return description.substring(0, Math.min(description.length(), MAX_DESCRIPTION_LENGTH));
+    }
+
+    public static List<String> limitTags(List<String> tags) {
+        if (tags == null) {
+            return null;
+        }
+
+        // Ensure tags list is mutable
+        List<String> mutableTags = new ArrayList<>(tags);
+
+        int tagCount = mutableTags.size();
+        if (tagCount == 0) {
+            return null;
+        }
+
+        // Remove tags over the limit
+        // This is cleaner than truncating, which results in malformed tags
+        // Also remove tags that are empty
+        Iterator iterator = mutableTags.iterator();
+        while (iterator.hasNext()) {
+            String tag = (String) iterator.next();
+            if (tag == null || tag.length() > MAX_TAG_LENGTH || tag.isEmpty()) {
+                iterator.remove();
+            }
+        }
+
+        // Limit the total number of tags
+        if (tagCount > MAX_TAGS_COUNT) {
+            mutableTags = mutableTags.subList(0, MAX_TAGS_COUNT);
+        }
+
+        return mutableTags;
+    }
+
 }

src/main/java/org/qortal/arbitrary/misc/Category.java

@@ -0,0 +1,81 @@
+package org.qortal.arbitrary.misc;
+
+public enum Category {
+    ART("Art and Design"),
+    AUTOMOTIVE("Automotive"),
+    BEAUTY("Beauty"),
+    BOOKS("Books and Reference"),
+    BUSINESS("Business"),
+    COMMUNICATIONS("Communications"),
+    CRYPTOCURRENCY("Cryptocurrency and Blockchain"),
+    CULTURE("Culture"),
+    DATING("Dating"),
+    DESIGN("Design"),
+    ENTERTAINMENT("Entertainment"),
+    EVENTS("Events"),
+    FAITH("Faith and Religion"),
+    FASHION("Fashion"),
+    FINANCE("Finance"),
+    FOOD("Food and Drink"),
+    GAMING("Gaming"),
+    GEOGRAPHY("Geography"),
+    HEALTH("Health"),
+    HISTORY("History"),
+    HOME("Home"),
+    KNOWLEDGE("Knowledge Share"),
+    LANGUAGE("Language"),
+    LIFESTYLE("Lifestyle"),
+    MANUFACTURING("Manufacturing"),
+    MAPS("Maps and Navigation"),
+    MUSIC("Music"),
+    NEWS("News"),
+    OTHER("Other"),
+    PETS("Pets"),
+    PHILOSOPHY("Philosophy"),
+    PHOTOGRAPHY("Photography"),
+    POLITICS("Politics"),
+    PRODUCE("Products and Services"),
+    PRODUCTIVITY("Productivity"),
+    PSYCHOLOGY("Psychology"),
+    QORTAL("Qortal"),
+    SCIENCE("Science"),
+    SELF_CARE("Self Care"),
+    SELF_SUFFICIENCY("Self-Sufficiency and Homesteading"),
+    SHOPPING("Shopping"),
+    SOCIAL("Social"),
+    SOFTWARE("Software"),
+    SPIRITUALITY("Spirituality"),
+    SPORTS("Sports"),
+    STORYTELLING("Storytelling"),
+    TECHNOLOGY("Technology"),
+    TOOLS("Tools"),
+    TRAVEL("Travel"),
+    UNCATEGORIZED("Uncategorized"),
+    VIDEO("Video"),
+    WEATHER("Weather");
+
+    private final String name;
+
+    Category(String name) {
+        this.name = name;
+    }
+
+    public String getName() {
+        return this.name;
+    }
+
+    /**
+     * Same as valueOf() but with fallback to UNCATEGORIZED if there's no match
+     * @param name
+     * @return a Category (using UNCATEGORIZED if no match found)
+     */
+    public static Category uncategorizedValueOf(String name) {
+        try {
+            return Category.valueOf(name);
+        }
+        catch (IllegalArgumentException e) {
+            return Category.UNCATEGORIZED;
+        }
+    }
+
+}

src/main/java/org/qortal/block/BlockChain.java

@@ -72,6 +72,11 @@ public class BlockChain {
 		transactionV5Timestamp;
 	}
 
+	// Custom transaction fees
+	@XmlJavaTypeAdapter(value = org.qortal.api.AmountTypeAdapter.class)
+	private long nameRegistrationUnitFee;
+	private long nameRegistrationUnitFeeTimestamp;
+
 	/** Map of which blockchain features are enabled when (height/timestamp) */
 	@XmlJavaTypeAdapter(StringLongMapXmlAdapter.class)
 	private Map<String, Long> featureTriggers;
@@ -301,6 +306,16 @@ public class BlockChain {
 		return this.maxBlockSize;
 	}
 
+	// Custom transaction fees
+	public long getNameRegistrationUnitFee() {
+		return this.nameRegistrationUnitFee;
+	}
+
+	public long getNameRegistrationUnitFeeTimestamp() {
+		// FUTURE: we could use a separate structure to indicate fee adjustments for different transaction types
+		return this.nameRegistrationUnitFeeTimestamp;
+	}
+
 	/** Returns true if approval-needing transaction types require a txGroupId other than NO_GROUP. */
 	public boolean getRequireGroupForApproval() {
 		return this.requireGroupForApproval;

src/main/java/org/qortal/controller/BlockMinter.java

@@ -343,6 +343,9 @@ public class BlockMinter extends Thread {
 						LOGGER.debug("Unable to check for a higher weight chain. Proceeding anyway...");
 					}
 
+					// Discard any uncommitted changes as a result of the higher weight chain detection
+					repository.discardChanges();
+
 					// Clear variables that track low weight blocks
 					parentSignatureForLastLowWeightBlock = null;
 					timeOfLastLowWeightBlock = null;

src/main/java/org/qortal/controller/Controller.java

@@ -105,6 +105,8 @@ public class Controller extends Thread {
 	private static final long LAST_SEEN_EXPIRY_PERIOD = (ONLINE_TIMESTAMP_MODULUS * 2) + (1 * 60 * 1000L);
 	/** How many (latest) blocks' worth of online accounts we cache */
 	private static final int MAX_BLOCKS_CACHED_ONLINE_ACCOUNTS = 2;
+	private static final long ONLINE_ACCOUNTS_V2_PEER_VERSION = 0x0300020000L;
+
 
 	private static volatile boolean isStopping = false;
 	private static BlockMinter blockMinter = null;
@@ -207,6 +209,15 @@ public class Controller extends Thread {
 		}
 		public GetArbitraryDataFileListMessageStats getArbitraryDataFileListMessageStats = new GetArbitraryDataFileListMessageStats();
 
+		public static class GetArbitraryMetadataMessageStats {
+			public AtomicLong requests = new AtomicLong();
+			public AtomicLong unknownFiles = new AtomicLong();
+
+			public GetArbitraryMetadataMessageStats() {
+			}
+		}
+		public GetArbitraryMetadataMessageStats getArbitraryMetadataMessageStats = new GetArbitraryMetadataMessageStats();
+
 		public AtomicLong latestBlocksCacheRefills = new AtomicLong();
 
 		public StatsSnapshot() {
@@ -515,7 +526,7 @@ public class Controller extends Thread {
 
 	@Override
 	public void run() {
-		Thread.currentThread().setName("Controller");
+		Thread.currentThread().setName("Qortal");
 
 		final long repositoryBackupInterval = Settings.getInstance().getRepositoryBackupInterval();
 		final long repositoryCheckpointInterval = Settings.getInstance().getRepositoryCheckpointInterval();
@@ -765,14 +776,14 @@ public class Controller extends Thread {
 				actionText = Translator.INSTANCE.translate("SysTray", "MINTING_ENABLED");
 				SysTray.getInstance().setTrayIcon(2);
 			}
-			else if (Synchronizer.getInstance().isSynchronizing()) {
-				actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
-				SysTray.getInstance().setTrayIcon(3);
-			}
 			else if (numberOfPeers < Settings.getInstance().getMinBlockchainPeers()) {
 				actionText = Translator.INSTANCE.translate("SysTray", "CONNECTING");
 				SysTray.getInstance().setTrayIcon(3);
 			}
+			else if (!this.isUpToDate()) {
+				actionText = String.format("%s - %d%%", Translator.INSTANCE.translate("SysTray", "SYNCHRONIZING_BLOCKCHAIN"), Synchronizer.getInstance().getSyncPercent());
+				SysTray.getInstance().setTrayIcon(3);
+			}
 			else {
 				actionText = Translator.INSTANCE.translate("SysTray", "MINTING_DISABLED");
 				SysTray.getInstance().setTrayIcon(4);
@@ -824,6 +835,16 @@ public class Controller extends Thread {
 
 	// Incoming transactions queue
 
+	private boolean incomingTransactionQueueContains(byte[] signature) {
+		synchronized (incomingTransactions) {
+			return incomingTransactions.stream().anyMatch(t -> Arrays.equals(t.getSignature(), signature));
+		}
+	}
+
+	private void removeIncomingTransaction(byte[] signature) {
+		incomingTransactions.removeIf(t -> Arrays.equals(t.getSignature(), signature));
+	}
+
 	private void processIncomingTransactionsQueue() {
 		if (this.incomingTransactions.size() == 0) {
 			// Don't bother locking if there are no new transactions to process
@@ -847,65 +868,73 @@ public class Controller extends Thread {
 		}
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
+			LOGGER.debug("Processing incoming transactions queue (size {})...", this.incomingTransactions.size());
+
+			// Take a copy of incomingTransactions so we can release the lock
+			List<TransactionData>incomingTransactionsCopy = new ArrayList<>(this.incomingTransactions);
 
 			// Iterate through incoming transactions list
-			synchronized (this.incomingTransactions) { // Required in order to safely iterate a synchronizedList()
-				Iterator iterator = this.incomingTransactions.iterator();
-				while (iterator.hasNext()) {
-					if (isStopping) {
-						return;
-					}
-
-					TransactionData transactionData = (TransactionData) iterator.next();
-					Transaction transaction = Transaction.fromData(repository, transactionData);
-
-					// Check signature
-					if (!transaction.isSignatureValid()) {
-						LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
-						iterator.remove();
-						continue;
-					}
-
-					ValidationResult validationResult = transaction.importAsUnconfirmed();
-
-					if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) {
-						LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature())));
-						iterator.remove();
-						continue;
-					}
-
-					if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) {
-						LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature())));
-						iterator.remove();
-						continue;
-					}
-
-					if (validationResult != ValidationResult.OK) {
-						final String signature58 = Base58.encode(transactionData.getSignature());
-						LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58));
-						Long now = NTP.getTime();
-						if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) {
-							Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL;
-							if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) {
-								// Use shorter recheck interval for expired transactions
-								expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL;
-							}
-							Long expiry = now + expiryLength;
-							LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
-							// Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it
-							invalidUnconfirmedTransactions.put(signature58, expiry);
-						}
-						iterator.remove();
-						continue;
-					}
-
-					LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
-					iterator.remove();
+			Iterator iterator = incomingTransactionsCopy.iterator();
+			while (iterator.hasNext()) {
+				if (isStopping) {
+					return;
 				}
+
+				if (Synchronizer.getInstance().isSyncRequestPending()) {
+					LOGGER.debug("Breaking out of transaction processing loop with {} remaining, because a sync request is pending", incomingTransactionsCopy.size());
+					return;
+				}
+
+				TransactionData transactionData = (TransactionData) iterator.next();
+				Transaction transaction = Transaction.fromData(repository, transactionData);
+
+				// Check signature
+				if (!transaction.isSignatureValid()) {
+					LOGGER.trace(() -> String.format("Ignoring %s transaction %s with invalid signature", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
+					removeIncomingTransaction(transactionData.getSignature());
+					continue;
+				}
+
+				ValidationResult validationResult = transaction.importAsUnconfirmed();
+
+				if (validationResult == ValidationResult.TRANSACTION_ALREADY_EXISTS) {
+					LOGGER.trace(() -> String.format("Ignoring existing transaction %s", Base58.encode(transactionData.getSignature())));
+					removeIncomingTransaction(transactionData.getSignature());
+					continue;
+				}
+
+				if (validationResult == ValidationResult.NO_BLOCKCHAIN_LOCK) {
+					LOGGER.trace(() -> String.format("Couldn't lock blockchain to import unconfirmed transaction", Base58.encode(transactionData.getSignature())));
+					removeIncomingTransaction(transactionData.getSignature());
+					continue;
+				}
+
+				if (validationResult != ValidationResult.OK) {
+					final String signature58 = Base58.encode(transactionData.getSignature());
+					LOGGER.trace(() -> String.format("Ignoring invalid (%s) %s transaction %s", validationResult.name(), transactionData.getType().name(), signature58));
+					Long now = NTP.getTime();
+					if (now != null && now - transactionData.getTimestamp() > INVALID_TRANSACTION_STALE_TIMEOUT) {
+						Long expiryLength = INVALID_TRANSACTION_RECHECK_INTERVAL;
+						if (validationResult == ValidationResult.TIMESTAMP_TOO_OLD) {
+							// Use shorter recheck interval for expired transactions
+							expiryLength = EXPIRED_TRANSACTION_RECHECK_INTERVAL;
+						}
+						Long expiry = now + expiryLength;
+						LOGGER.debug("Adding stale invalid transaction {} to invalidUnconfirmedTransactions...", signature58);
+						// Invalid, unconfirmed transaction has become stale - add to invalidUnconfirmedTransactions so that we don't keep requesting it
+						invalidUnconfirmedTransactions.put(signature58, expiry);
+					}
+					removeIncomingTransaction(transactionData.getSignature());
+					continue;
+				}
+
+				LOGGER.debug(() -> String.format("Imported %s transaction %s", transactionData.getType().name(), Base58.encode(transactionData.getSignature())));
+				removeIncomingTransaction(transactionData.getSignature());
 			}
 		} catch (DataException e) {
 			LOGGER.error(String.format("Repository issue while processing incoming transactions", e));
 		} finally {
+			LOGGER.debug("Finished processing incoming transactions queue");
 			blockchainLock.unlock();
 		}
 	}
@@ -1264,6 +1293,14 @@ public class Controller extends Thread {
 				onNetworkOnlineAccountsMessage(peer, message);
 				break;
 
+			case GET_ONLINE_ACCOUNTS_V2:
+				onNetworkGetOnlineAccountsV2Message(peer, message);
+				break;
+
+			case ONLINE_ACCOUNTS_V2:
+				onNetworkOnlineAccountsV2Message(peer, message);
+				break;
+
 			case GET_ARBITRARY_DATA:
 				// Not currently supported
 				break;
@@ -1284,6 +1321,21 @@ public class Controller extends Thread {
 				ArbitraryDataManager.getInstance().onNetworkArbitrarySignaturesMessage(peer, message);
 				break;
 
+			case GET_ARBITRARY_METADATA:
+				ArbitraryMetadataManager.getInstance().onNetworkGetArbitraryMetadataMessage(peer, message);
+				break;
+
+			case ARBITRARY_METADATA:
+				ArbitraryMetadataManager.getInstance().onNetworkArbitraryMetadataMessage(peer, message);
+				break;
+
+			case GET_TRADE_PRESENCES:
+				TradeBot.getInstance().onGetTradePresencesMessage(peer, message);
+				break;
+
+			case TRADE_PRESENCES:
+				TradeBot.getInstance().onTradePresencesMessage(peer, message);
+
 			default:
 				LOGGER.debug(() -> String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer));
 				break;
@@ -1594,6 +1646,12 @@ public class Controller extends Thread {
 					continue;
 				}
 
+				// Ignore if this transaction is in the queue
+				if (incomingTransactionQueueContains(signature)) {
+					LOGGER.trace(() -> String.format("Ignoring existing queued transaction %s from peer %s", Base58.encode(signature), peer));
+					continue;
+				}
+
 				// Do we have it already? (Before requesting transaction data itself)
 				if (repository.getTransactionRepository().exists(signature)) {
 					LOGGER.trace(() -> String.format("Ignoring existing transaction %s from peer %s", Base58.encode(signature), peer));
@@ -1663,6 +1721,53 @@ public class Controller extends Thread {
 		}
 	}
 
+	private void onNetworkGetOnlineAccountsV2Message(Peer peer, Message message) {
+		GetOnlineAccountsV2Message getOnlineAccountsMessage = (GetOnlineAccountsV2Message) message;
+
+		List<OnlineAccountData> excludeAccounts = getOnlineAccountsMessage.getOnlineAccounts();
+
+		// Send online accounts info, excluding entries with matching timestamp & public key from excludeAccounts
+		List<OnlineAccountData> accountsToSend;
+		synchronized (this.onlineAccounts) {
+			accountsToSend = new ArrayList<>(this.onlineAccounts);
+		}
+
+		Iterator<OnlineAccountData> iterator = accountsToSend.iterator();
+
+		SEND_ITERATOR:
+		while (iterator.hasNext()) {
+			OnlineAccountData onlineAccountData = iterator.next();
+
+			for (int i = 0; i < excludeAccounts.size(); ++i) {
+				OnlineAccountData excludeAccountData = excludeAccounts.get(i);
+
+				if (onlineAccountData.getTimestamp() == excludeAccountData.getTimestamp() && Arrays.equals(onlineAccountData.getPublicKey(), excludeAccountData.getPublicKey())) {
+					iterator.remove();
+					continue SEND_ITERATOR;
+				}
+			}
+		}
+
+		Message onlineAccountsMessage = new OnlineAccountsV2Message(accountsToSend);
+		peer.sendMessage(onlineAccountsMessage);
+
+		LOGGER.trace(() -> String.format("Sent %d of our %d online accounts to %s", accountsToSend.size(), this.onlineAccounts.size(), peer));
+	}
+
+	private void onNetworkOnlineAccountsV2Message(Peer peer, Message message) {
+		OnlineAccountsV2Message onlineAccountsMessage = (OnlineAccountsV2Message) message;
+
+		List<OnlineAccountData> peersOnlineAccounts = onlineAccountsMessage.getOnlineAccounts();
+		LOGGER.trace(() -> String.format("Received %d online accounts from %s", peersOnlineAccounts.size(), peer));
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (OnlineAccountData onlineAccountData : peersOnlineAccounts)
+				this.verifyAndAddAccount(repository, onlineAccountData);
+		} catch (DataException e) {
+			LOGGER.error(String.format("Repository issue while verifying online accounts from peer %s", peer), e);
+		}
+	}
+
 	// Utilities
 
 	private void verifyAndAddAccount(Repository repository, OnlineAccountData onlineAccountData) throws DataException {
@@ -1774,11 +1879,17 @@ public class Controller extends Thread {
 
 		// Request data from other peers?
 		if ((this.onlineAccountsTasksTimestamp % ONLINE_ACCOUNTS_BROADCAST_INTERVAL) < ONLINE_ACCOUNTS_TASKS_INTERVAL) {
-			Message message;
+			List<OnlineAccountData> safeOnlineAccounts;
 			synchronized (this.onlineAccounts) {
-				message = new GetOnlineAccountsMessage(this.onlineAccounts);
+				safeOnlineAccounts = new ArrayList<>(this.onlineAccounts);
 			}
-			Network.getInstance().broadcast(peer -> message);
+
+			Message messageV1 = new GetOnlineAccountsMessage(safeOnlineAccounts);
+			Message messageV2 = new GetOnlineAccountsV2Message(safeOnlineAccounts);
+
+			Network.getInstance().broadcast(peer ->
+				peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
+			);
 		}
 
 		// Refresh our online accounts signatures?
@@ -1870,8 +1981,12 @@ public class Controller extends Thread {
 			if (!hasInfoChanged)
 				return;
 
-			Message message = new OnlineAccountsMessage(ourOnlineAccounts);
-			Network.getInstance().broadcast(peer -> message);
+		Message messageV1 = new OnlineAccountsMessage(ourOnlineAccounts);
+		Message messageV2 = new OnlineAccountsV2Message(ourOnlineAccounts);
+
+		Network.getInstance().broadcast(peer ->
+				peer.getPeersVersion() >= ONLINE_ACCOUNTS_V2_PEER_VERSION ? messageV2 : messageV1
+		);
 
 			LOGGER.trace(() -> String.format("Broadcasted %d online account%s with timestamp %d", ourOnlineAccounts.size(), (ourOnlineAccounts.size() != 1 ? "s" : ""), onlineAccountsTimestamp));
 		}
@@ -1957,10 +2072,13 @@ public class Controller extends Thread {
 		return peers;
 	}
 
-	/** Returns whether we think our node has up-to-date blockchain based on our info about other peers. */
-	public boolean isUpToDate() {
+	/**
+	 * Returns whether we think our node has up-to-date blockchain based on our info about other peers.
+	 * @param minLatestBlockTimestamp - the minimum block timestamp to be considered recent
+	 * @return boolean - whether our node's blockchain is up to date or not
+	 */
+	public boolean isUpToDate(Long minLatestBlockTimestamp) {
 		// Do we even have a vaguely recent block?
-		final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
 		if (minLatestBlockTimestamp == null)
 			return false;
 
@@ -1986,6 +2104,16 @@ public class Controller extends Thread {
 		return !peers.isEmpty();
 	}
 
+	/**
+	 * Returns whether we think our node has up-to-date blockchain based on our info about other peers.
+	 * Uses the default minLatestBlockTimestamp value.
+	 * @return boolean - whether our node's blockchain is up to date or not
+	 */
+	public boolean isUpToDate() {
+		final Long minLatestBlockTimestamp = getMinimumLatestBlockTimestamp();
+		return this.isUpToDate(minLatestBlockTimestamp);
+	}
+
 	/** Returns minimum block timestamp for block to be considered 'recent', or <tt>null</tt> if NTP not synced. */
 	public static Long getMinimumLatestBlockTimestamp() {
 		Long now = NTP.getTime();

src/main/java/org/qortal/controller/Synchronizer.java

@@ -22,6 +22,8 @@ import org.qortal.data.block.CommonBlockData;
 import org.qortal.data.network.PeerChainTipData;
 import org.qortal.data.transaction.RewardShareTransactionData;
 import org.qortal.data.transaction.TransactionData;
+import org.qortal.event.Event;
+import org.qortal.event.EventBus;
 import org.qortal.network.Network;
 import org.qortal.network.Peer;
 import org.qortal.network.message.BlockMessage;
@@ -83,6 +85,7 @@ public class Synchronizer extends Thread {
 	private volatile int syncPercent = 0;
 
 	private static volatile boolean requestSync = false;
+	private boolean syncRequestPending = false;
 
 	// Keep track of invalid blocks so that we don't keep trying to sync them
 	private Map<String, Long> invalidBlockSignatures = Collections.synchronizedMap(new HashMap<>());
@@ -95,6 +98,24 @@ public class Synchronizer extends Thread {
 		OK, NOTHING_TO_DO, GENESIS_ONLY, NO_COMMON_BLOCK, TOO_DIVERGENT, NO_REPLY, INFERIOR_CHAIN, INVALID_DATA, NO_BLOCKCHAIN_LOCK, REPOSITORY_ISSUE, SHUTTING_DOWN;
 	}
 
+	public static class NewChainTipEvent implements Event {
+		private final BlockData priorChainTip;
+		private final BlockData newChainTip;
+
+		public NewChainTipEvent(BlockData priorChainTip, BlockData newChainTip) {
+			this.priorChainTip = priorChainTip;
+			this.newChainTip = newChainTip;
+		}
+
+		public BlockData getPriorChainTip() {
+			return this.priorChainTip;
+		}
+
+		public BlockData getNewChainTip() {
+			return this.newChainTip;
+		}
+	}
+
 	// Constructors
 
 	private Synchronizer() {
@@ -111,6 +132,8 @@ public class Synchronizer extends Thread {
 
 	@Override
 	public void run() {
+		Thread.currentThread().setName("Synchronizer");
+
 		try {
 			while (running && !Controller.isStopping()) {
 				Thread.sleep(1000);
@@ -122,6 +145,8 @@ public class Synchronizer extends Thread {
 						// Something went wrong, so try again next time
 						requestSync = true;
 					}
+					// Remember that we have a pending sync request if this attempt failed
+					syncRequestPending = !success;
 				}
 			}
 		} catch (InterruptedException e) {
@@ -142,6 +167,10 @@ public class Synchronizer extends Thread {
 		return this.isSynchronizing;
 	}
 
+	public boolean isSyncRequestPending() {
+		return this.syncRequestPending;
+	}
+
 	public Integer getSyncPercent() {
 		synchronized (this.syncLock) {
 			return this.isSynchronizing ? this.syncPercent : null;
@@ -329,6 +358,8 @@ public class Synchronizer extends Thread {
 
 				Network network = Network.getInstance();
 				network.broadcast(broadcastPeer -> network.buildHeightMessage(broadcastPeer, newChainTip));
+
+				EventBus.INSTANCE.notify(new NewChainTipEvent(priorChainTip, newChainTip));
 			}
 
 			return syncResult;

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@@ -37,11 +37,16 @@ public class ArbitraryDataBuildManager extends Thread {
 
     @Override
     public void run() {
+        Thread.currentThread().setName("Arbitrary Data Build Manager");
+
         try {
             // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
             // This can be expanded to have multiple threads processing the build queue when needed
-            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
-            arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+            int threadCount = 5;
+            ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(threadCount);
+            for (int i = 0; i < threadCount; i++) {
+                arbitraryDataBuildExecutor.execute(new ArbitraryDataBuilderThread());
+            }
 
             while (!isStopping) {
                 // Nothing to do yet
@@ -101,7 +106,7 @@ public class ArbitraryDataBuildManager extends Thread {
             return true;
         }
 
-        LOGGER.info("Added {} to build queue", queueItem);
+        log(queueItem, String.format("Added %s to build queue", queueItem));
 
         // Added to queue
         return true;
@@ -149,7 +154,7 @@ public class ArbitraryDataBuildManager extends Thread {
             return true;
         }
 
-        LOGGER.info("Added {} to failed builds list", queueItem);
+        log(queueItem, String.format("Added %s to failed builds list", queueItem));
 
         // Added to queue
         return true;
@@ -182,4 +187,17 @@ public class ArbitraryDataBuildManager extends Thread {
     public boolean getBuildInProgress() {
         return this.buildInProgress;
     }
+
+    private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
+        if (queueItem == null) {
+            return;
+        }
+
+        if (queueItem.isHighPriority()) {
+            LOGGER.info(message);
+        }
+        else {
+            LOGGER.debug(message);
+        }
+    }
 }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@@ -9,6 +9,7 @@ import org.qortal.repository.DataException;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
+import java.util.Comparator;
 import java.util.Map;
 
 
@@ -27,7 +28,7 @@ public class ArbitraryDataBuilderThread implements Runnable {
 
         while (!Controller.isStopping()) {
             try {
-                Thread.sleep(1000);
+                Thread.sleep(100);
 
                 if (buildManager.arbitraryDataBuildQueue == null) {
                     continue;
@@ -36,48 +37,57 @@ public class ArbitraryDataBuilderThread implements Runnable {
                     continue;
                 }
 
-                // Find resources that are queued for building
-                Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
-                        .entrySet().stream()
-                        .filter(e -> e.getValue().isQueued())
-                        .findFirst().orElse(null);
-
-                if (next == null) {
-                    continue;
-                }
-
                 Long now = NTP.getTime();
                 if (now == null) {
                     continue;
                 }
 
-                ArbitraryDataBuildQueueItem queueItem = next.getValue();
+                ArbitraryDataBuildQueueItem queueItem = null;
 
-                if (queueItem == null) {
-                    this.removeFromQueue(queueItem);
+                // Find resources that are queued for building (sorted by highest priority first)
+                synchronized (buildManager.arbitraryDataBuildQueue) {
+                    Map.Entry<String, ArbitraryDataBuildQueueItem> next = buildManager.arbitraryDataBuildQueue
+                            .entrySet().stream()
+                            .filter(e -> e.getValue().isQueued())
+                            .sorted(Comparator.comparing(item -> item.getValue().getPriority()))
+                            .reduce((first, second) -> second).orElse(null);
+
+                    if (next == null) {
+                        continue;
+                    }
+
+                    queueItem = next.getValue();
+
+                    if (queueItem == null) {
+                        this.removeFromQueue(queueItem);
+                        continue;
+                    }
+
+                    // Ignore builds that have failed recently
+                    if (buildManager.isInFailedBuildsList(queueItem)) {
+                        this.removeFromQueue(queueItem);
+                        continue;
+                    }
+
+                    // Set the start timestamp, to prevent other threads from building it at the same time
+                    queueItem.prepareForBuild();
                 }
 
-                // Ignore builds that have failed recently
-                if (buildManager.isInFailedBuildsList(queueItem)) {
-                    continue;
-                }
-
-
                 try {
                     // Perform the build
-                    LOGGER.info("Building {}...", queueItem);
+                    log(queueItem, String.format("Building %s... priority: %d", queueItem, queueItem.getPriority()));
                     queueItem.build();
                     this.removeFromQueue(queueItem);
-                    LOGGER.info("Finished building {}", queueItem);
+                    log(queueItem, String.format("Finished building %s", queueItem));
 
                 } catch (MissingDataException e) {
-                    LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage());
+                    log(queueItem, String.format("Missing data for %s: %s", queueItem, e.getMessage()));
                     queueItem.setFailed(true);
                     this.removeFromQueue(queueItem);
                     // Don't add to the failed builds list, as we may want to retry sooner
 
                 } catch (IOException | DataException | RuntimeException e) {
-                    LOGGER.info("Error building {}: {}", queueItem, e.getMessage());
+                    log(queueItem, String.format("Error building %s: %s", queueItem, e.getMessage()));
                     // Something went wrong - so remove it from the queue, and add to failed builds list
                     queueItem.setFailed(true);
                     buildManager.addToFailedBuildsList(queueItem);
@@ -96,4 +106,17 @@ public class ArbitraryDataBuilderThread implements Runnable {
         }
         ArbitraryDataBuildManager.getInstance().arbitraryDataBuildQueue.remove(queueItem.getUniqueKey());
     }
+
+    private void log(ArbitraryDataBuildQueueItem queueItem, String message) {
+        if (queueItem == null) {
+            return;
+        }
+
+        if (queueItem.isHighPriority()) {
+            LOGGER.info(message);
+        }
+        else {
+            LOGGER.debug(message);
+        }
+    }
 }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -222,7 +222,11 @@ public class ArbitraryDataCleanupManager extends Thread {
 				try (final Repository repository = RepositoryManager.getRepository()) {
 
 					// Check if there are any hosted files that don't have matching transactions
-					this.checkForExpiredTransactions(repository);
+					// UPDATE: This has been disabled for now as it was deleting valid transactions
+					// and causing chunks to go missing on the network. If ever re-enabled, we MUST
+					// ensure that original copies of data aren't deleted, and that sufficient time
+					// is allowed (ideally several hours) before treating a transaction as missing.
+					// this.checkForExpiredTransactions(repository);
 
 					// Delete additional data at random if we're over our storage limit
 					// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileListManager.java

@@ -29,6 +29,7 @@ public class ArbitraryDataFileListManager {
 
     private static ArbitraryDataFileListManager instance;
 
+    private static String MIN_PEER_VERSION_FOR_FILE_LIST_STATS = "3.2.0";
 
     /**
      * Map of recent incoming requests for ARBITRARY transaction data file lists.
@@ -58,9 +59,9 @@ public class ArbitraryDataFileListManager {
 
 
     /** Maximum number of seconds that a file list relay request is able to exist on the network */
-    private static long RELAY_REQUEST_MAX_DURATION = 5000L;
+    public static long RELAY_REQUEST_MAX_DURATION = 5000L;
     /** Maximum number of hops that a file list relay request is allowed to make */
-    private static int RELAY_REQUEST_MAX_HOPS = 4;
+    public static int RELAY_REQUEST_MAX_HOPS = 4;
 
 
     private ArbitraryDataFileListManager() {
@@ -266,18 +267,16 @@ public class ArbitraryDataFileListManager {
         List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
         List<byte[]> missingHashes = null;
 
-//        // TODO: uncomment after GetArbitraryDataFileListMessage updates are deployed
-//        // Find hashes that we are missing
-//        try {
-//            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
-//            arbitraryDataFile.setMetadataHash(metadataHash);
-//            missingHashes = arbitraryDataFile.missingHashes();
-//        } catch (DataException e) {
-//            // Leave missingHashes as null, so that all hashes are requested
-//        }
-//        int hashCount = missingHashes != null ? missingHashes.size() : 0;
+        // Find hashes that we are missing
+        try {
+            ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
+            arbitraryDataFile.setMetadataHash(metadataHash);
+            missingHashes = arbitraryDataFile.missingHashes();
+        } catch (DataException e) {
+            // Leave missingHashes as null, so that all hashes are requested
+        }
+        int hashCount = missingHashes != null ? missingHashes.size() : 0;
 
-        int hashCount = 0;
         LOGGER.debug(String.format("Sending data file list request for signature %s with %d hashes to %d peers...", signature58, hashCount, handshakedPeers.size()));
 
         // Build request
@@ -405,6 +404,13 @@ public class ArbitraryDataFileListManager {
         ArbitraryDataFileListMessage arbitraryDataFileListMessage = (ArbitraryDataFileListMessage) message;
         LOGGER.debug("Received hash list from peer {} with {} hashes", peer, arbitraryDataFileListMessage.getHashes().size());
 
+        if (LOGGER.isDebugEnabled() && arbitraryDataFileListMessage.getRequestTime() != null) {
+            long totalRequestTime = NTP.getTime() - arbitraryDataFileListMessage.getRequestTime();
+            LOGGER.debug("totalRequestTime: {}, requestHops: {}, peerAddress: {}, isRelayPossible: {}",
+                    totalRequestTime, arbitraryDataFileListMessage.getRequestHops(),
+                    arbitraryDataFileListMessage.getPeerAddress(), arbitraryDataFileListMessage.isRelayPossible());
+        }
+
         // Do we have a pending request for this data?
         Triple<String, Peer, Long> request = arbitraryDataFileListRequests.get(message.getId());
         if (request == null || request.getA() == null) {
@@ -474,12 +480,26 @@ public class ArbitraryDataFileListManager {
             if (!isBlocked) {
                 Peer requestingPeer = request.getB();
                 if (requestingPeer != null) {
+                    Long requestTime = arbitraryDataFileListMessage.getRequestTime();
+                    Integer requestHops = arbitraryDataFileListMessage.getRequestHops();
+
                     // Add each hash to our local mapping so we know who to ask later
                     Long now = NTP.getTime();
                     for (byte[] hash : hashes) {
                         String hash58 = Base58.encode(hash);
-                        ArbitraryRelayInfo relayMap = new ArbitraryRelayInfo(hash58, signature58, peer, now);
-                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayMap);
+                        ArbitraryRelayInfo relayInfo = new ArbitraryRelayInfo(hash58, signature58, peer, now, requestTime, requestHops);
+                        ArbitraryDataFileManager.getInstance().addToRelayMap(relayInfo);
+                    }
+
+                    // Bump requestHops if it exists
+                    if (requestHops != null) {
+                        arbitraryDataFileListMessage.setRequestHops(++requestHops);
+                    }
+
+                    // Remove optional parameters if the requesting peer doesn't support it yet
+                    // A message with less statistical data is better than no message at all
+                    if (!requestingPeer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                        arbitraryDataFileListMessage.removeOptionalStats();
                     }
 
                     // Forward to requesting peer
@@ -509,7 +529,7 @@ public class ArbitraryDataFileListManager {
 
         // If we've seen this request recently, then ignore
         if (arbitraryDataFileListRequests.putIfAbsent(message.getId(), newEntry) != null) {
-            LOGGER.debug("Ignoring hash list request from peer {} for signature {}", peer, signature58);
+            LOGGER.trace("Ignoring hash list request from peer {} for signature {}", peer, signature58);
             return;
         }
 
@@ -518,6 +538,7 @@ public class ArbitraryDataFileListManager {
         List<byte[]> hashes = new ArrayList<>();
         ArbitraryTransactionData transactionData = null;
         boolean allChunksExist = false;
+        boolean hasMetadata = false;
 
         try (final Repository repository = RepositoryManager.getRepository()) {
 
@@ -542,6 +563,7 @@ public class ArbitraryDataFileListManager {
                         // Add the metadata file
                         if (arbitraryDataFile.getMetadataHash() != null) {
                             requestedHashes.add(arbitraryDataFile.getMetadataHash());
+                            hasMetadata = true;
                         }
 
                         // Add the chunk hashes
@@ -574,6 +596,12 @@ public class ArbitraryDataFileListManager {
             LOGGER.error(String.format("Repository issue while fetching arbitrary file list for peer %s", peer), e);
         }
 
+        // If the only file we have is the metadata then we shouldn't respond. Most nodes will already have that,
+        // or can use the separate metadata protocol to fetch it. This should greatly reduce network spam.
+        if (hasMetadata && hashes.size() == 1) {
+            hashes.clear();
+        }
+
         // We should only respond if we have at least one hash
         if (hashes.size() > 0) {
 
@@ -584,8 +612,17 @@ public class ArbitraryDataFileListManager {
                 arbitraryDataFileListRequests.put(message.getId(), newEntry);
             }
 
-            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature, hashes);
+            String ourAddress = Network.getInstance().getOurExternalIpAddress();
+            ArbitraryDataFileListMessage arbitraryDataFileListMessage = new ArbitraryDataFileListMessage(signature,
+                    hashes, NTP.getTime(), 0, ourAddress, true);
             arbitraryDataFileListMessage.setId(message.getId());
+
+            // Remove optional parameters if the requesting peer doesn't support it yet
+            // A message with less statistical data is better than no message at all
+            if (!peer.isAtLeastVersion(MIN_PEER_VERSION_FOR_FILE_LIST_STATS)) {
+                arbitraryDataFileListMessage.removeOptionalStats();
+            }
+
             if (!peer.sendMessage(arbitraryDataFileListMessage)) {
                 LOGGER.debug("Couldn't send list of hashes");
                 peer.disconnect("failed to send list of hashes");

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -37,7 +37,7 @@ public class ArbitraryDataFileManager extends Thread {
     /**
      * Map to keep track of our in progress (outgoing) arbitrary data file requests
      */
-    private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
+    public Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
 
     /**
      * Map to keep track of hashes that we might need to relay
@@ -148,7 +148,7 @@ public class ArbitraryDataFileManager extends Thread {
                     }
                 }
                 else {
-                    LOGGER.trace("Already requesting data file {} for signature {}", arbitraryDataFile, Base58.encode(signature));
+                    LOGGER.trace("Already requesting data file {} for signature {} from peer {}", arbitraryDataFile, Base58.encode(signature), peer);
                 }
             }
             else {
@@ -240,16 +240,7 @@ public class ArbitraryDataFileManager extends Thread {
                 ArbitraryDataFile dataFile = arbitraryDataFileMessage.getArbitraryDataFile();
 
                 // Keep trying to delete the data until it is deleted, or we reach 10 attempts
-                for (int i=0; i<10; i++) {
-                    if (dataFile.delete()) {
-                        break;
-                    }
-                    try {
-                        Thread.sleep(1000L);
-                    } catch (InterruptedException e) {
-                        // Fall through to exit method
-                    }
-                }
+                dataFile.delete(10);
             }
         }
 
@@ -401,17 +392,44 @@ public class ArbitraryDataFileManager extends Thread {
         }
     }
 
+    private ArbitraryRelayInfo getOptimalRelayInfoEntryForHash(String hash58) {
+        LOGGER.trace("Fetching relay info for hash: {}", hash58);
+        List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
+        if (relayInfoList != null && !relayInfoList.isEmpty()) {
+
+            // Remove any with null requestHops
+            relayInfoList.removeIf(r -> r.getRequestHops() == null);
+
+            // If list is now empty, then just return one at random
+            if (relayInfoList.isEmpty()) {
+                return this.getRandomRelayInfoEntryForHash(hash58);
+            }
+
+            // Sort by number of hops (lowest first)
+            relayInfoList.sort(Comparator.comparingInt(ArbitraryRelayInfo::getRequestHops));
+
+            // FUTURE: secondary sort by requestTime?
+
+            ArbitraryRelayInfo relayInfo = relayInfoList.get(0);
+
+            LOGGER.trace("Returning optimal relay info for hash: {} (requestHops {})", hash58, relayInfo.getRequestHops());
+            return relayInfo;
+        }
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
+        return null;
+    }
+
     private ArbitraryRelayInfo getRandomRelayInfoEntryForHash(String hash58) {
-        LOGGER.info("Fetching random relay info for hash: {}", hash58);
+        LOGGER.trace("Fetching random relay info for hash: {}", hash58);
         List<ArbitraryRelayInfo> relayInfoList = this.getRelayInfoListForHash(hash58);
         if (relayInfoList != null && !relayInfoList.isEmpty()) {
 
             // Pick random item
             int index = new SecureRandom().nextInt(relayInfoList.size());
-            LOGGER.info("Returning random relay info for hash: {} (index {})", hash58, index);
+            LOGGER.trace("Returning random relay info for hash: {} (index {})", hash58, index);
             return relayInfoList.get(index);
         }
-        LOGGER.info("No relay info exists for hash: {}", hash58);
+        LOGGER.trace("No relay info exists for hash: {}", hash58);
         return null;
     }
 
@@ -451,7 +469,7 @@ public class ArbitraryDataFileManager extends Thread {
 
         try {
             ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash, signature);
-            ArbitraryRelayInfo relayInfo = this.getRandomRelayInfoEntryForHash(hash58);
+            ArbitraryRelayInfo relayInfo = this.getOptimalRelayInfoEntryForHash(hash58);
 
             if (arbitraryDataFile.exists()) {
                 LOGGER.trace("Hash {} exists", hash58);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileRequestThread.java

@@ -31,8 +31,6 @@ public class ArbitraryDataFileRequestThread implements Runnable {
 
         try {
             while (!Controller.isStopping()) {
-                Thread.sleep(1000);
-
                 Long now = NTP.getTime();
                 this.processFileHashes(now);
             }
@@ -41,67 +39,72 @@ public class ArbitraryDataFileRequestThread implements Runnable {
         }
     }
 
-    private void processFileHashes(Long now) {
-        if (Controller.isStopping()) {
+    private void processFileHashes(Long now) throws InterruptedException {
+		if (Controller.isStopping()) {
             return;
         }
 
-        try (final Repository repository = RepositoryManager.getRepository()) {
-            ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
+        ArbitraryDataFileManager arbitraryDataFileManager = ArbitraryDataFileManager.getInstance();
+        String signature58 = null;
+        String hash58 = null;
+        Peer peer = null;
+        boolean shouldProcess = false;
 
-            ArbitraryTransactionData arbitraryTransactionData = null;
-            byte[] signature = null;
-            byte[] hash = null;
-            Peer peer = null;
-            boolean shouldProcess = false;
-
-            synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
-                Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
-                while (iterator.hasNext()) {
-                    if (Controller.isStopping()) {
-                        return;
-                    }
-
-                    Map.Entry entry = (Map.Entry) iterator.next();
-                    if (entry == null || entry.getKey() == null || entry.getValue() == null) {
-                        iterator.remove();
-                        continue;
-                    }
-
-                    String hash58 = (String) entry.getKey();
-                    Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
-                    if (value == null) {
-                        iterator.remove();
-                        continue;
-                    }
-
-                    peer = value.getA();
-                    String signature58 = value.getB();
-                    Long timestamp = value.getC();
-
-                    if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
-                        // Ignore - to be deleted
-                        iterator.remove();
-                        continue;
-                    }
-
-                    hash = Base58.decode(hash58);
-                    signature = Base58.decode(signature58);
-
-                    // We want to process this file
-                    shouldProcess = true;
-                    iterator.remove();
-                    break;
+        synchronized (arbitraryDataFileManager.arbitraryDataFileHashResponses) {
+            Iterator iterator = arbitraryDataFileManager.arbitraryDataFileHashResponses.entrySet().iterator();
+            while (iterator.hasNext()) {
+                if (Controller.isStopping()) {
+                    return;
                 }
-            }
 
-            if (!shouldProcess) {
-                // Nothing to do
-                return;
-            }
+                Map.Entry entry = (Map.Entry) iterator.next();
+                if (entry == null || entry.getKey() == null || entry.getValue() == null) {
+                    iterator.remove();
+                    continue;
+                }
 
-            // Fetch the transaction data
-            arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+                hash58 = (String) entry.getKey();
+                Triple<Peer, String, Long> value = (Triple<Peer, String, Long>) entry.getValue();
+                if (value == null) {
+                    iterator.remove();
+                    continue;
+                }
+
+                peer = value.getA();
+                signature58 = value.getB();
+                Long timestamp = value.getC();
+
+                if (now - timestamp >= ArbitraryDataManager.ARBITRARY_RELAY_TIMEOUT || signature58 == null || peer == null) {
+                    // Ignore - to be deleted
+                    iterator.remove();
+                    continue;
+                }
+
+                // Skip if already requesting, but don't remove, as we might want to retry later
+                if (arbitraryDataFileManager.arbitraryDataFileRequests.containsKey(hash58)) {
+                    // Already requesting - leave this attempt for later
+                    continue;
+                }
+
+                // We want to process this file
+                shouldProcess = true;
+                iterator.remove();
+                break;
+            }
+        }
+
+        if (!shouldProcess) {
+            // Nothing to do
+            Thread.sleep(1000L);
+            return;
+        }
+
+        byte[] hash = Base58.decode(hash58);
+        byte[] signature = Base58.decode(signature58);
+
+        // Fetch the transaction data
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
             if (arbitraryTransactionData == null) {
                 return;
             }
@@ -110,7 +113,6 @@ public class ArbitraryDataFileRequestThread implements Runnable {
                 return;
             }
 
-            String hash58 = Base58.encode(hash);
             LOGGER.debug("Fetching file {} from peer {} via request thread...", hash58, peer);
             arbitraryDataFileManager.fetchArbitraryDataFiles(repository, peer, signature, arbitraryTransactionData, Arrays.asList(hash));
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -9,6 +9,7 @@ import org.apache.logging.log4j.Logger;
 import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
 import org.qortal.data.network.ArbitraryPeerData;
@@ -38,7 +39,7 @@ public class ArbitraryDataManager extends Thread {
 	private int powDifficulty = 14; // Must not be final, as unit tests need to reduce this value
 
 	/** Request timeout when transferring arbitrary data */
-	public static final long ARBITRARY_REQUEST_TIMEOUT = 10 * 1000L; // ms
+	public static final long ARBITRARY_REQUEST_TIMEOUT = 12 * 1000L; // ms
 
 	/** Maximum time to hold information about an in-progress relay */
 	public static final long ARBITRARY_RELAY_TIMEOUT = 60 * 1000L; // ms
@@ -80,6 +81,9 @@ public class ArbitraryDataManager extends Thread {
 		Thread.currentThread().setName("Arbitrary Data Manager");
 
 		try {
+			// Wait for node to finish starting up and making connections
+			Thread.sleep(2 * 60 * 1000L);
+
 			while (!isStopping) {
 				Thread.sleep(2000);
 
@@ -99,6 +103,10 @@ public class ArbitraryDataManager extends Thread {
 					continue;
 				}
 
+				// Fetch metadata
+				// Disabled for now. TODO: re-enable later.
+				// this.fetchAllMetadata();
+
 				// Fetch data according to storage policy
 				switch (Settings.getInstance().getStoragePolicy()) {
 					case FOLLOWED:
@@ -222,6 +230,83 @@ public class ArbitraryDataManager extends Thread {
 		}
 	}
 
+	private void fetchAllMetadata() {
+		ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+
+		// Paginate queries when fetching arbitrary transactions
+		final int limit = 100;
+		int offset = 0;
+
+		while (!isStopping) {
+
+			// Any arbitrary transactions we want to fetch data for?
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, null, ConfirmationStatus.BOTH, limit, offset, true);
+				// LOGGER.trace("Found {} arbitrary transactions at offset: {}, limit: {}", signatures.size(), offset, limit);
+				if (signatures == null || signatures.isEmpty()) {
+					offset = 0;
+					break;
+				}
+				offset += limit;
+
+				// Loop through signatures and remove ones we don't need to process
+				Iterator iterator = signatures.iterator();
+				while (iterator.hasNext()) {
+					byte[] signature = (byte[]) iterator.next();
+
+					ArbitraryTransaction arbitraryTransaction = fetchTransaction(repository, signature);
+					if (arbitraryTransaction == null) {
+						// Best not to process this one
+						iterator.remove();
+						continue;
+					}
+					ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
+
+					// Skip transactions that are blocked
+					if (storageManager.isBlocked(arbitraryTransactionData)) {
+						iterator.remove();
+						continue;
+					}
+
+					// Remove transactions that we already have local data for
+					if (hasLocalMetadata(arbitraryTransaction)) {
+						iterator.remove();
+						continue;
+					}
+				}
+
+				if (signatures.isEmpty()) {
+					continue;
+				}
+
+				// Pick one at random
+				final int index = new Random().nextInt(signatures.size());
+				byte[] signature = signatures.get(index);
+
+				if (signature == null) {
+					continue;
+				}
+
+				// Check to see if we have had a more recent PUT
+				ArbitraryTransactionData arbitraryTransactionData = ArbitraryTransactionUtils.fetchTransactionData(repository, signature);
+				boolean hasMoreRecentPutTransaction = ArbitraryTransactionUtils.hasMoreRecentPutTransaction(repository, arbitraryTransactionData);
+				if (hasMoreRecentPutTransaction) {
+					// There is a more recent PUT transaction than the one we are currently processing.
+					// When a PUT is issued, it replaces any layers that would have been there before.
+					// Therefore any data relating to this older transaction is no longer needed and we
+					// shouldn't fetch it from the network.
+					continue;
+				}
+
+				// Ask our connected peers if they have metadata for this signature
+				fetchMetadata(arbitraryTransactionData);
+
+			} catch (DataException e) {
+				LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
+			}
+		}
+	}
+
 	private ArbitraryTransaction fetchTransaction(final Repository repository, byte[] signature) {
 		try {
 			TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
@@ -241,16 +326,42 @@ public class ArbitraryDataManager extends Thread {
 
 		} catch (DataException e) {
 			LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
-			return true;
+			return true; // Assume true for now, to avoid network spam on error
 		}
 	}
 
+	private boolean hasLocalMetadata(ArbitraryTransaction arbitraryTransaction) {
+		try {
+			ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
+			byte[] signature = arbitraryTransactionData.getSignature();
+			byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+			ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+
+			return metadataFile.exists();
+
+		} catch (DataException e) {
+			LOGGER.error("Repository issue when checking arbitrary transaction's metadata is local", e);
+			return true; // Assume true for now, to avoid network spam on error
+		}
+	}
 
 	// Entrypoint to request new data from peers
 	public boolean fetchData(ArbitraryTransactionData arbitraryTransactionData) {
 		return ArbitraryDataFileListManager.getInstance().fetchArbitraryDataFileList(arbitraryTransactionData);
 	}
 
+	// Entrypoint to request new metadata from peers
+	public ArbitraryDataTransactionMetadata fetchMetadata(ArbitraryTransactionData arbitraryTransactionData) {
+
+		ArbitraryDataResource resource = new ArbitraryDataResource(
+				arbitraryTransactionData.getName(),
+				ArbitraryDataFile.ResourceIdType.NAME,
+				arbitraryTransactionData.getService(),
+				arbitraryTransactionData.getIdentifier()
+		);
+		return ArbitraryMetadataManager.getInstance().fetchMetadata(resource, true);
+	}
+
 
 	// Useful methods used by other parts of the app
 
@@ -275,6 +386,9 @@ public class ArbitraryDataManager extends Thread {
 
 		// Cleanup file request caches
 		ArbitraryDataFileManager.getInstance().cleanupRequestCache(now);
+
+		// Clean up metadata request caches
+		ArbitraryMetadataManager.getInstance().cleanupRequestCache(now);
 	}
 
 	public boolean isResourceCached(ArbitraryDataResource resource) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataRenderManager.java

@@ -32,7 +32,7 @@ public class ArbitraryDataRenderManager extends Thread {
 
     @Override
     public void run() {
-        Thread.currentThread().setName("Arbitrary Data Manager");
+        Thread.currentThread().setName("Arbitrary Data Render Manager");
 
         try {
             while (!isStopping) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -16,6 +16,7 @@ import org.qortal.utils.FilesystemUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -46,6 +47,9 @@ public class ArbitraryDataStorageManager extends Thread {
 
     private List<ArbitraryTransactionData> hostedTransactions;
 
+    private String searchQuery;
+    private List<ArbitraryTransactionData> searchResultsTransactions;
+
     private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
 
     /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
@@ -225,6 +229,16 @@ public class ArbitraryDataStorageManager extends Thread {
         }
     }
 
+    /**
+     * Check if data relating to a transaction is blocked by this node.
+     *
+     * @param arbitraryTransactionData - the transaction
+     * @return boolean - whether the resource is blocked or not
+     */
+    public boolean isBlocked(ArbitraryTransactionData arbitraryTransactionData) {
+        return isNameBlocked(arbitraryTransactionData.getName());
+    }
+
     private boolean isDataTypeAllowed(ArbitraryTransactionData arbitraryTransactionData) {
         byte[] secret = arbitraryTransactionData.getSecret();
         boolean hasSecret = (secret != null && secret.length == 32);
@@ -257,14 +271,8 @@ public class ArbitraryDataStorageManager extends Thread {
     }
 
 
-    // Hosted data
-
-    public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset) {
-        // Load from cache if we can, to avoid disk reads
-        if (this.hostedTransactions != null) {
-            return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
-        }
-
+    public List<ArbitraryTransactionData> loadAllHostedTransactions(Repository repository) {
+        
         List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
 
         // Find all hosted paths
@@ -285,7 +293,21 @@ public class ArbitraryDataStorageManager extends Thread {
                 if (transactionData == null || transactionData.getType() != Transaction.TransactionType.ARBITRARY) {
                     continue;
                 }
-                arbitraryTransactionDataList.add((ArbitraryTransactionData) transactionData);
+                ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+
+                // Make sure to exclude metadata-only resources
+                if (arbitraryTransactionData.getMetadataHash() != null) {
+                    if (contents.length == 1) {
+                        String metadataHash58 = Base58.encode(arbitraryTransactionData.getMetadataHash());
+                        if (Objects.equals(metadataHash58, contents[0])) {
+                            // We only have the metadata file for this resource, not the actual data, so exclude it
+                            continue;
+                        }
+                    }
+                }
+
+                // Found some data matching a transaction, so add it to the list
+                arbitraryTransactionDataList.add(arbitraryTransactionData);
 
             } catch (DataException e) {
                 continue;
@@ -295,10 +317,69 @@ public class ArbitraryDataStorageManager extends Thread {
         // Sort by newest first
         arbitraryTransactionDataList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
 
-        // Update cache
-        this.hostedTransactions = arbitraryTransactionDataList;
+        return arbitraryTransactionDataList;
+    }
+    // Hosted data
 
-        return ArbitraryTransactionUtils.limitOffsetTransactions(arbitraryTransactionDataList, limit, offset);
+    public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository, Integer limit, Integer offset) {
+        // Load from cache if we can, to avoid disk reads
+
+        if (this.hostedTransactions != null) {
+            return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
+        }
+
+        this.hostedTransactions = this.loadAllHostedTransactions(repository);
+
+        return ArbitraryTransactionUtils.limitOffsetTransactions(this.hostedTransactions, limit, offset);
+    }
+    
+    /**
+     * searchHostedTransactions
+     * Allow to run a query against hosted data names and return matches if there are any
+     * @param repository
+     * @param query
+     * @param limit
+     * @param offset
+     * @return
+     */
+
+    public List<ArbitraryTransactionData> searchHostedTransactions(Repository repository, String query, Integer limit, Integer offset) {
+        // Load from results cache if we can (results that exists for the same query), to avoid disk reads
+        if (this.searchResultsTransactions != null && this.searchQuery.equals(query.toLowerCase())) {
+            return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset);
+        }
+
+        // Using cache if we can, to avoid disk reads
+        if (this.hostedTransactions == null) {
+            this.hostedTransactions = this.loadAllHostedTransactions(repository);
+        }
+
+        this.searchQuery = query.toLowerCase(); //set the searchQuery so that it can be checked on the next call
+
+        List<ArbitraryTransactionData> searchResultsList = new ArrayList<>();
+
+        // Loop through cached hostedTransactions
+        for (ArbitraryTransactionData atd : this.hostedTransactions) {
+            try {
+               if (atd.getName() != null && atd.getName().toLowerCase().contains(this.searchQuery)) {
+                   searchResultsList.add(atd);
+               }
+               else if (atd.getIdentifier() != null && atd.getIdentifier().toLowerCase().contains(this.searchQuery)) {
+                   searchResultsList.add(atd);
+               }
+
+            } catch (Exception e) {
+                continue;
+            }
+        }
+
+        // Sort by newest first
+        searchResultsList.sort(Comparator.comparingLong(ArbitraryTransactionData::getTimestamp).reversed());
+
+        // Update cache
+        this.searchResultsTransactions = searchResultsList;
+
+        return ArbitraryTransactionUtils.limitOffsetTransactions(this.searchResultsTransactions, limit, offset);
     }
 
     /**
@@ -322,7 +403,7 @@ public class ArbitraryDataStorageManager extends Thread {
                             && path.getFileName().toString().length() > 32)
                     .collect(Collectors.toList());
         }
-        catch (IOException e) {
+        catch (IOException | UncheckedIOException e) {
             LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
         }
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -0,0 +1,456 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.controller.Controller;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.TransactionData;
+import org.qortal.network.Network;
+import org.qortal.network.Peer;
+import org.qortal.network.message.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
+import org.qortal.utils.Triple;
+
+import java.io.IOException;
+import java.util.*;
+
+import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.RELAY_REQUEST_MAX_DURATION;
+import static org.qortal.controller.arbitrary.ArbitraryDataFileListManager.RELAY_REQUEST_MAX_HOPS;
+
+public class ArbitraryMetadataManager {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryMetadataManager.class);
+
+    private static ArbitraryMetadataManager instance;
+
+    /**
+     * Map of recent incoming requests for ARBITRARY transaction metadata.
+     * <p>
+     * Key is original request's message ID<br>
+     * Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
+     * <p>
+     * If peer is null then either:<br>
+     * <ul>
+     * <li>we are the original requesting peer</li>
+     * <li>we have already sent data payload to original requesting peer.</li>
+     * </ul>
+     * If signature is null then we have already received the file list and either:<br>
+     * <ul>
+     * <li>we are the original requesting peer and have processed it</li>
+     * <li>we have forwarded the metadata</li>
+     * </ul>
+     */
+    public Map<Integer, Triple<String, Peer, Long>> arbitraryMetadataRequests = Collections.synchronizedMap(new HashMap<>());
+
+    /**
+     * Map to keep track of in progress arbitrary metadata requests
+     * Key: string - the signature encoded in base58
+     * Value: Triple<networkBroadcastCount, directPeerRequestCount, lastAttemptTimestamp>
+     */
+    private Map<String, Triple<Integer, Integer, Long>> arbitraryMetadataSignatureRequests = Collections.synchronizedMap(new HashMap<>());
+
+
+    private ArbitraryMetadataManager() {
+    }
+
+    public static ArbitraryMetadataManager getInstance() {
+        if (instance == null)
+            instance = new ArbitraryMetadataManager();
+
+        return instance;
+    }
+
+    public void cleanupRequestCache(Long now) {
+        if (now == null) {
+            return;
+        }
+        final long requestMinimumTimestamp = now - ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT;
+        arbitraryMetadataRequests.entrySet().removeIf(entry -> entry.getValue().getC() == null || entry.getValue().getC() < requestMinimumTimestamp);
+    }
+
+
+    public ArbitraryDataTransactionMetadata fetchMetadata(ArbitraryDataResource arbitraryDataResource, boolean useRateLimiter) {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Find latest transaction
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(arbitraryDataResource.getResourceId(), arbitraryDataResource.getService(),
+                            null, arbitraryDataResource.getIdentifier());
+
+            if (latestTransaction != null) {
+                byte[] signature = latestTransaction.getSignature();
+                byte[] metadataHash = latestTransaction.getMetadataHash();
+                if (metadataHash == null) {
+                    // This resource doesn't have metadata
+                    throw new IllegalArgumentException("This resource doesn't have metadata");
+                }
+
+                ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                if (!metadataFile.exists()) {
+                    // Request from network
+                    this.fetchArbitraryMetadata(latestTransaction, useRateLimiter);
+                }
+
+                // Now check again as it may have been downloaded above
+                if (metadataFile.exists()) {
+                    // Use local copy
+                    ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+                    transactionMetadata.read();
+                    return transactionMetadata;
+                }
+            }
+
+        } catch (DataException | IOException e) {
+            LOGGER.error("Repository issue when fetching arbitrary transaction metadata", e);
+        }
+
+        return null;
+    }
+
+
+    // Request metadata from network
+
+    public byte[] fetchArbitraryMetadata(ArbitraryTransactionData arbitraryTransactionData, boolean useRateLimiter) {
+        byte[] metadataHash = arbitraryTransactionData.getMetadataHash();
+        if (metadataHash == null) {
+            return null;
+        }
+
+        byte[] signature = arbitraryTransactionData.getSignature();
+        String signature58 = Base58.encode(signature);
+
+        // Require an NTP sync
+        Long now = NTP.getTime();
+        if (now == null) {
+            return null;
+        }
+
+        // If we've already tried too many times in a short space of time, make sure to give up
+        if (useRateLimiter && !this.shouldMakeMetadataRequestForSignature(signature58)) {
+            LOGGER.trace("Skipping metadata request for signature {} due to rate limit", signature58);
+            return null;
+        }
+        this.addToSignatureRequests(signature58, true, false);
+
+        List<Peer> handshakedPeers = Network.getInstance().getHandshakedPeers();
+        LOGGER.debug(String.format("Sending metadata request for signature %s to %d peers...", signature58, handshakedPeers.size()));
+
+        // Build request
+        Message getArbitraryMetadataMessage = new GetArbitraryMetadataMessage(signature, now, 0);
+
+        // Save our request into requests map
+        Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
+
+        // Assign random ID to this message
+        int id;
+        do {
+            id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
+
+            // Put queue into map (keyed by message ID) so we can poll for a response
+            // If putIfAbsent() doesn't return null, then this ID is already taken
+        } while (arbitraryMetadataRequests.put(id, requestEntry) != null);
+        getArbitraryMetadataMessage.setId(id);
+
+        // Broadcast request
+        Network.getInstance().broadcast(peer -> getArbitraryMetadataMessage);
+
+        // Poll to see if data has arrived
+        final long singleWait = 100;
+        long totalWait = 0;
+        while (totalWait < ArbitraryDataManager.ARBITRARY_REQUEST_TIMEOUT) {
+            try {
+                Thread.sleep(singleWait);
+            } catch (InterruptedException e) {
+                break;
+            }
+
+            requestEntry = arbitraryMetadataRequests.get(id);
+            if (requestEntry == null)
+                return null;
+
+            if (requestEntry.getA() == null)
+                break;
+
+            totalWait += singleWait;
+        }
+
+        try {
+            ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+            return metadataFile.getBytes();
+        } catch (DataException e) {
+            // Do nothing
+        }
+
+        return null;
+    }
+
+
+    // Track metadata lookups by signature
+
+    private boolean shouldMakeMetadataRequestForSignature(String signature58) {
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        // Extract the components
+        Integer networkBroadcastCount = request.getA();
+        // Integer directPeerRequestCount = request.getB();
+        Long lastAttemptTimestamp = request.getC();
+
+        if (lastAttemptTimestamp == null) {
+            // Not attempted yet
+            return true;
+        }
+
+        long timeSinceLastAttempt = NTP.getTime() - lastAttemptTimestamp;
+
+        // Allow a second attempt after 15 seconds, and another after 30 seconds
+        if (timeSinceLastAttempt > 15 * 1000L) {
+            // We haven't tried for at least 15 seconds
+
+            if (networkBroadcastCount < 3) {
+                // We've made less than 3 total attempts
+                return true;
+            }
+        }
+
+        // Then allow another 5 attempts, each 5 minutes apart
+        if (timeSinceLastAttempt > 5 * 60 * 1000L) {
+            // We haven't tried for at least 5 minutes
+
+            if (networkBroadcastCount < 5) {
+                // We've made less than 5 total attempts
+                return true;
+            }
+        }
+
+        // From then on, only try once every 24 hours, to reduce network spam
+        if (timeSinceLastAttempt > 24 * 60 * 60 * 1000L) {
+            // We haven't tried for at least 24 hours
+            return true;
+        }
+
+        return false;
+    }
+
+    public boolean isSignatureRateLimited(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        return !this.shouldMakeMetadataRequestForSignature(signature58);
+    }
+
+    public long lastRequestForSignature(byte[] signature) {
+        String signature58 = Base58.encode(signature);
+        Triple<Integer, Integer, Long> request = arbitraryMetadataSignatureRequests.get(signature58);
+
+        if (request == null) {
+            // Not attempted yet
+            return 0;
+        }
+
+        // Extract the components
+        Long lastAttemptTimestamp = request.getC();
+        if (lastAttemptTimestamp != null) {
+            return  lastAttemptTimestamp;
+        }
+        return 0;
+    }
+
+    public void addToSignatureRequests(String signature58, boolean incrementNetworkRequests, boolean incrementPeerRequests) {
+        Triple<Integer, Integer, Long> request  = arbitraryMetadataSignatureRequests.get(signature58);
+        Long now = NTP.getTime();
+
+        if (request == null) {
+            // No entry yet
+            Triple<Integer, Integer, Long> newRequest = new Triple<>(0, 0, now);
+            arbitraryMetadataSignatureRequests.put(signature58, newRequest);
+        }
+        else {
+            // There is an existing entry
+            if (incrementNetworkRequests) {
+                request.setA(request.getA() + 1);
+            }
+            if (incrementPeerRequests) {
+                request.setB(request.getB() + 1);
+            }
+            request.setC(now);
+            arbitraryMetadataSignatureRequests.put(signature58, request);
+        }
+    }
+
+    public void removeFromSignatureRequests(String signature58) {
+        arbitraryMetadataSignatureRequests.remove(signature58);
+    }
+
+
+    // Network handlers
+
+    public void onNetworkArbitraryMetadataMessage(Peer peer, Message message) {
+        // Don't process if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        ArbitraryMetadataMessage arbitraryMetadataMessage = (ArbitraryMetadataMessage) message;
+        LOGGER.debug("Received metadata from peer {}", peer);
+
+        // Do we have a pending request for this data?
+        Triple<String, Peer, Long> request = arbitraryMetadataRequests.get(message.getId());
+        if (request == null || request.getA() == null) {
+            return;
+        }
+        boolean isRelayRequest = (request.getB() != null);
+
+        // Does this message's signature match what we're expecting?
+        byte[] signature = arbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        if (!request.getA().equals(signature58)) {
+            return;
+        }
+
+        // Update requests map to reflect that we've received all chunks
+        Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
+        arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+        ArbitraryTransactionData arbitraryTransactionData = null;
+
+        // Forwarding
+        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+
+            // Get transaction info
+            try (final Repository repository = RepositoryManager.getRepository()) {
+                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+                if (!(transactionData instanceof ArbitraryTransactionData))
+                    return;
+                arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
+            } catch (DataException e) {
+                LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
+            }
+
+            // Check if the name is blocked
+            boolean isBlocked = (arbitraryTransactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(arbitraryTransactionData.getName()));
+            if (!isBlocked) {
+                Peer requestingPeer = request.getB();
+                if (requestingPeer != null) {
+
+                    // Forward to requesting peer
+                    LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                    if (!requestingPeer.sendMessage(arbitraryMetadataMessage)) {
+                        requestingPeer.disconnect("failed to forward arbitrary metadata");
+                    }
+                }
+            }
+        }
+    }
+
+    public void onNetworkGetArbitraryMetadataMessage(Peer peer, Message message) {
+        // Don't respond if QDN is disabled
+        if (!Settings.getInstance().isQdnEnabled()) {
+            return;
+        }
+
+        Controller.getInstance().stats.getArbitraryMetadataMessageStats.requests.incrementAndGet();
+
+        GetArbitraryMetadataMessage getArbitraryMetadataMessage = (GetArbitraryMetadataMessage) message;
+        byte[] signature = getArbitraryMetadataMessage.getSignature();
+        String signature58 = Base58.encode(signature);
+        Long now = NTP.getTime();
+        Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, now);
+
+        // If we've seen this request recently, then ignore
+        if (arbitraryMetadataRequests.putIfAbsent(message.getId(), newEntry) != null) {
+            LOGGER.debug("Ignoring metadata request from peer {} for signature {}", peer, signature58);
+            return;
+        }
+
+        LOGGER.debug("Received metadata request from peer {} for signature {}", peer, signature58);
+
+        ArbitraryTransactionData transactionData = null;
+        ArbitraryDataFile metadataFile = null;
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Firstly we need to lookup this file on chain to get its metadata hash
+            transactionData = (ArbitraryTransactionData)repository.getTransactionRepository().fromSignature(signature);
+            if (transactionData instanceof ArbitraryTransactionData) {
+
+                // Check if we're even allowed to serve metadata for this transaction
+                if (ArbitraryDataStorageManager.getInstance().canStoreData(transactionData)) {
+
+                    byte[] metadataHash = transactionData.getMetadataHash();
+                    if (metadataHash != null) {
+
+                        // Load metadata file
+                        metadataFile = ArbitraryDataFile.fromHash(metadataHash, signature);
+                    }
+                }
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while fetching arbitrary metadata for peer %s", peer), e);
+        }
+
+        // We should only respond if we have the metadata file
+        if (metadataFile != null && metadataFile.exists()) {
+
+            // We have the metadata file, so update requests map to reflect that we've sent it
+            newEntry = new Triple<>(null, null, now);
+            arbitraryMetadataRequests.put(message.getId(), newEntry);
+
+            ArbitraryMetadataMessage arbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, metadataFile);
+            arbitraryMetadataMessage.setId(message.getId());
+            if (!peer.sendMessage(arbitraryMetadataMessage)) {
+                LOGGER.debug("Couldn't send metadata");
+                peer.disconnect("failed to send metadata");
+                return;
+            }
+            LOGGER.debug("Sent metadata");
+
+            // Nothing left to do, so return to prevent any unnecessary forwarding from occurring
+            LOGGER.debug("No need for any forwarding because metadata request is fully served");
+            return;
+
+        }
+
+        // We may need to forward this request on
+        boolean isBlocked = (transactionData == null || ArbitraryDataStorageManager.getInstance().isNameBlocked(transactionData.getName()));
+        if (Settings.getInstance().isRelayModeEnabled() && !isBlocked) {
+            // In relay mode - so ask our other peers if they have it
+
+            long requestTime = getArbitraryMetadataMessage.getRequestTime();
+            int requestHops = getArbitraryMetadataMessage.getRequestHops();
+            getArbitraryMetadataMessage.setRequestHops(++requestHops);
+            long totalRequestTime = now - requestTime;
+
+            if (totalRequestTime < RELAY_REQUEST_MAX_DURATION) {
+                // Relay request hasn't timed out yet, so can potentially be rebroadcast
+                if (requestHops < RELAY_REQUEST_MAX_HOPS) {
+                    // Relay request hasn't reached the maximum number of hops yet, so can be rebroadcast
+
+                    LOGGER.debug("Rebroadcasting metadata request from peer {} for signature {} to our other peers... totalRequestTime: {}, requestHops: {}", peer, Base58.encode(signature), totalRequestTime, requestHops);
+                    Network.getInstance().broadcast(
+                            broadcastPeer -> broadcastPeer == peer ||
+                                    Objects.equals(broadcastPeer.getPeerData().getAddress().getHost(), peer.getPeerData().getAddress().getHost())
+                                    ? null : getArbitraryMetadataMessage);
+
+                }
+                else {
+                    // This relay request has reached the maximum number of allowed hops
+                }
+            }
+            else {
+                // This relay request has timed out
+            }
+        }
+    }
+
+}

src/main/java/org/qortal/controller/tradebot/TradeBot.java

@@ -2,41 +2,38 @@ package org.qortal.controller.tradebot;
 
 import java.awt.TrayIcon.MessageType;
 import java.security.SecureRandom;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-import java.util.concurrent.locks.ReentrantLock;
+import java.util.*;
+import java.util.function.Supplier;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.util.Supplier;
 import org.bitcoinj.core.ECKey;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.api.model.crosschain.TradeBotCreateRequest;
 import org.qortal.controller.Controller;
+import org.qortal.controller.Synchronizer;
 import org.qortal.controller.tradebot.AcctTradeBot.ResponseResult;
 import org.qortal.crosschain.*;
+import org.qortal.crypto.Crypto;
 import org.qortal.data.at.ATData;
 import org.qortal.data.crosschain.CrossChainTradeData;
 import org.qortal.data.crosschain.TradeBotData;
-import org.qortal.data.transaction.BaseTransactionData;
-import org.qortal.data.transaction.PresenceTransactionData;
+import org.qortal.data.network.TradePresenceData;
 import org.qortal.event.Event;
 import org.qortal.event.EventBus;
 import org.qortal.event.Listener;
-import org.qortal.group.Group;
 import org.qortal.gui.SysTray;
+import org.qortal.network.Network;
+import org.qortal.network.Peer;
+import org.qortal.network.message.GetTradePresencesMessage;
+import org.qortal.network.message.Message;
+import org.qortal.network.message.TradePresencesMessage;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.repository.hsqldb.HSQLDBImportExport;
 import org.qortal.settings.Settings;
-import org.qortal.transaction.PresenceTransaction;
-import org.qortal.transaction.PresenceTransaction.PresenceType;
-import org.qortal.transaction.Transaction.ValidationResult;
-import org.qortal.transform.transaction.TransactionTransformer;
+import org.qortal.utils.ByteArray;
 import org.qortal.utils.NTP;
 
 import com.google.common.primitives.Longs;
@@ -56,6 +53,15 @@ public class TradeBot implements Listener {
 	private static final Logger LOGGER = LogManager.getLogger(TradeBot.class);
 	private static final Random RANDOM = new SecureRandom();
 
+	/** Maximum lifetime of trade presence timestamp. 30 mins in ms. */
+	private static final long PRESENCE_LIFETIME = 30 * 60 * 1000L;
+	/** How soon before expiry of our own trade presence timestamp that we want to trigger renewal. 5 mins in ms. */
+	private static final long EARLY_RENEWAL_PERIOD = 5 * 60 * 1000L;
+	/** Trade presence timestamps are rounded up to this nearest interval. Bigger values improve grouping of entries in [GET_]TRADE_PRESENCES network messages. 15 mins in ms. */
+	private static final long EXPIRY_ROUNDING = 15 * 60 * 1000L;
+	/** How often we want to broadcast our list of all known trade presences to peers. 5 mins in ms. */
+	private static final long PRESENCE_BROADCAST_INTERVAL = 5 * 60 * 1000L;
+
 	public interface StateNameAndValueSupplier {
 		public String getState();
 		public int getStateValue();
@@ -73,6 +79,18 @@ public class TradeBot implements Listener {
 		}
 	}
 
+	public static class TradePresenceEvent implements Event {
+		private final TradePresenceData tradePresenceData;
+
+		public TradePresenceEvent(TradePresenceData tradePresenceData) {
+			this.tradePresenceData = tradePresenceData;
+		}
+
+		public TradePresenceData getTradePresenceData() {
+			return this.tradePresenceData;
+		}
+	}
+
 	private static final Map<Class<? extends ACCT>, Supplier<AcctTradeBot>> acctTradeBotSuppliers = new HashMap<>();
 	static {
 		acctTradeBotSuppliers.put(BitcoinACCTv1.class, BitcoinACCTv1TradeBot::getInstance);
@@ -86,7 +104,12 @@ public class TradeBot implements Listener {
 
 	private static TradeBot instance;
 
-	private final Map<String, Long> presenceTimestampsByAtAddress = Collections.synchronizedMap(new HashMap<>());
+	private final Map<ByteArray, Long> ourTradePresenceTimestampsByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private final List<TradePresenceData> pendingTradePresences = Collections.synchronizedList(new ArrayList<>());
+
+	private final Map<ByteArray, TradePresenceData> allTradePresencesByPubkey = Collections.synchronizedMap(new HashMap<>());
+	private Map<ByteArray, TradePresenceData> safeAllTradePresencesByPubkey = Collections.emptyMap();
+	private long nextTradePresenceBroadcastTimestamp = 0L;
 
 	private TradeBot() {
 		EventBus.INSTANCE.addListener(event -> TradeBot.getInstance().listen(event));
@@ -213,10 +236,12 @@ public class TradeBot implements Listener {
 
 	@Override
 	public void listen(Event event) {
-		if (!(event instanceof Controller.NewBlockEvent))
+		if (!(event instanceof Synchronizer.NewChainTipEvent))
 			return;
 
 		synchronized (this) {
+			expireOldPresenceTimestamps();
+
 			List<TradeBotData> allTradeBotData;
 
 			try (final Repository repository = RepositoryManager.getRepository()) {
@@ -247,6 +272,8 @@ public class TradeBot implements Listener {
 				} catch (ForeignBlockchainException e) {
 					LOGGER.warn(() -> String.format("Foreign blockchain issue processing trade-bot entry for AT %s: %s", tradeBotData.getAtAddress(), e.getMessage()));
 				}
+
+			broadcastPresenceTimestamps();
 		}
 	}
 
@@ -324,6 +351,33 @@ public class TradeBot implements Listener {
 	}
 
 	// PRESENCE-related
+
+	public Collection<TradePresenceData> getAllTradePresences() {
+		return this.safeAllTradePresencesByPubkey.values();
+	}
+
+	/** Trade presence timestamps expire in the 'future' so any that reach 'now' have expired and are removed. */
+	private void expireOldPresenceTimestamps() {
+		long now = NTP.getTime();
+
+		int allRemovedCount = 0;
+		synchronized (this.allTradePresencesByPubkey) {
+			int preRemoveCount = this.allTradePresencesByPubkey.size();
+			this.allTradePresencesByPubkey.values().removeIf(tradePresenceData -> tradePresenceData.getTimestamp() <= now);
+			allRemovedCount = this.allTradePresencesByPubkey.size() - preRemoveCount;
+		}
+
+		int ourRemovedCount = 0;
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			int preRemoveCount = this.ourTradePresenceTimestampsByPubkey.size();
+			this.ourTradePresenceTimestampsByPubkey.values().removeIf(timestamp -> timestamp < now);
+			ourRemovedCount = this.ourTradePresenceTimestampsByPubkey.size() - preRemoveCount;
+		}
+
+		if (allRemovedCount > 0)
+			LOGGER.debug("Removed {} expired trade presences, of which {} ours", allRemovedCount, ourRemovedCount);
+	}
+
 	/*package*/ void updatePresence(Repository repository, TradeBotData tradeBotData, CrossChainTradeData tradeData)
 			throws DataException {
 		String atAddress = tradeBotData.getAtAddress();
@@ -332,44 +386,292 @@ public class TradeBot implements Listener {
 		String signerAddress = tradeNativeAccount.getAddress();
 
 		/*
-		 * There's no point in Alice trying to build a PRESENCE transaction
-		 * for an AT that isn't locked to her, as other peers won't be able
-		 * to validate the PRESENCE transaction as signing public key won't
-		 * be visible.
-		 */
-		if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress))
-			// Signer is neither Bob, nor Alice, or trade not yet locked to Alice
+		* There's no point in Alice trying to broadcast presence for an AT that isn't locked to her,
+		* as other peers won't be able to verify as signing public key isn't yet in the AT's data segment.
+		*/
+		if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+			// Signer is neither Bob, nor trade locked to Alice
+			LOGGER.trace("Can't provide trade presence for our AT {} as it's not yet locked to Alice", atAddress);
 			return;
+		}
 
 		long now = NTP.getTime();
-		long threshold = now - PresenceType.TRADE_BOT.getLifetime();
+		long newExpiry = generateExpiry(now);
+		ByteArray pubkeyByteArray = ByteArray.of(tradeNativeAccount.getPublicKey());
 
-		long timestamp = presenceTimestampsByAtAddress.compute(atAddress, (k, v) -> (v == null || v < threshold) ? now : v);
+		// If map entry's timestamp is missing, or within early renewal period, use the new expiry - otherwise use existing timestamp.
+		synchronized (this.ourTradePresenceTimestampsByPubkey) {
+			Long currentTimestamp = this.ourTradePresenceTimestampsByPubkey.get(pubkeyByteArray);
 
-		// If timestamp hasn't been updated then nothing to do
-		if (timestamp != now)
+			if (currentTimestamp != null && currentTimestamp - now > EARLY_RENEWAL_PERIOD) {
+				// timestamp still good
+				LOGGER.trace("Current trade presence timestamp {} still good for our trade {}", currentTimestamp, atAddress);
+				return;
+			}
+
+			this.ourTradePresenceTimestampsByPubkey.put(pubkeyByteArray, newExpiry);
+		}
+
+		// Create signature
+		byte[] signature = tradeNativeAccount.sign(Longs.toByteArray(newExpiry));
+
+		// Add new trade presence to queue to be broadcast around network
+		TradePresenceData tradePresenceData = new TradePresenceData(newExpiry, tradeNativeAccount.getPublicKey(), signature, atAddress);
+		this.pendingTradePresences.add(tradePresenceData);
+
+		this.allTradePresencesByPubkey.put(pubkeyByteArray, tradePresenceData);
+		rebuildSafeAllTradePresences();
+
+		LOGGER.trace("New trade presence timestamp {} for our trade {}", newExpiry, atAddress);
+
+		EventBus.INSTANCE.notify(new TradePresenceEvent(tradePresenceData));
+	}
+
+	private void rebuildSafeAllTradePresences() {
+		synchronized (this.allTradePresencesByPubkey) {
+			// Collect into a *new* unmodifiable map.
+			this.safeAllTradePresencesByPubkey = Map.copyOf(this.allTradePresencesByPubkey);
+		}
+	}
+
+	private void broadcastPresenceTimestamps() {
+		// If we have new trade presences that are pending broadcast, send those as a priority
+		if (!this.pendingTradePresences.isEmpty()) {
+			// Create a copy for Network to safely use in another thread
+			List<TradePresenceData> safeTradePresences;
+			synchronized (this.pendingTradePresences) {
+				safeTradePresences = List.copyOf(this.pendingTradePresences);
+				this.pendingTradePresences.clear();
+			}
+
+			LOGGER.debug("Broadcasting {} new trade presences", safeTradePresences.size());
+
+			TradePresencesMessage tradePresencesMessage = new TradePresencesMessage(safeTradePresences);
+			Network.getInstance().broadcast(peer -> tradePresencesMessage);
+
+			return;
+		}
+
+		// As we have no new trade presences, check whether it's time to do a general broadcast
+		Long now = NTP.getTime();
+		if (now == null || now < nextTradePresenceBroadcastTimestamp)
 			return;
 
-		int txGroupId = Group.NO_GROUP;
-		byte[] reference = new byte[TransactionTransformer.SIGNATURE_LENGTH];
-		byte[] creatorPublicKey = tradeNativeAccount.getPublicKey();
-		long fee = 0L;
+		nextTradePresenceBroadcastTimestamp = now + PRESENCE_BROADCAST_INTERVAL;
 
-		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, null);
+		List<TradePresenceData> safeTradePresences = List.copyOf(this.safeAllTradePresencesByPubkey.values());
 
-		int nonce = 0;
-		byte[] timestampSignature = tradeNativeAccount.sign(Longs.toByteArray(timestamp));
+		if (safeTradePresences.isEmpty())
+			return;
 
-		PresenceTransactionData transactionData = new PresenceTransactionData(baseTransactionData, nonce, PresenceType.TRADE_BOT, timestampSignature);
+		LOGGER.debug("Broadcasting all {} known trade presences. Next broadcast timestamp: {}",
+				safeTradePresences.size(), nextTradePresenceBroadcastTimestamp
+		);
 
-		PresenceTransaction presenceTransaction = new PresenceTransaction(repository, transactionData);
-		presenceTransaction.computeNonce();
+		GetTradePresencesMessage getTradePresencesMessage = new GetTradePresencesMessage(safeTradePresences);
+		Network.getInstance().broadcast(peer -> getTradePresencesMessage);
+	}
 
-		presenceTransaction.sign(tradeNativeAccount);
+	// Network message processing
 
-		ValidationResult result = presenceTransaction.importAsUnconfirmed();
-		if (result != ValidationResult.OK)
-			LOGGER.debug(() -> String.format("Unable to build trade-bot PRESENCE transaction for %s: %s", tradeBotData.getAtAddress(), result.name()));
+	public void onGetTradePresencesMessage(Peer peer, Message message) {
+		GetTradePresencesMessage getTradePresencesMessage = (GetTradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = getTradePresencesMessage.getTradePresences();
+
+		// Create mutable copy from safe snapshot
+		Map<ByteArray, TradePresenceData> entriesUnknownToPeer = new HashMap<>(this.safeAllTradePresencesByPubkey);
+		int knownCount = entriesUnknownToPeer.size();
+
+		for (TradePresenceData peersTradePresence : peersTradePresences) {
+			ByteArray pubkeyByteArray = ByteArray.of(peersTradePresence.getPublicKey());
+
+			TradePresenceData ourEntry = entriesUnknownToPeer.get(pubkeyByteArray);
+
+			if (ourEntry != null && ourEntry.getTimestamp() == peersTradePresence.getTimestamp())
+				entriesUnknownToPeer.remove(pubkeyByteArray);
+		}
+
+		if (entriesUnknownToPeer.isEmpty())
+			return;
+
+		LOGGER.debug("Sending {} trade presences to peer {} after excluding their {} from known {}",
+				entriesUnknownToPeer.size(), peer, peersTradePresences.size(), knownCount
+		);
+
+		// Send complement to peer
+		List<TradePresenceData> safeTradePresences = List.copyOf(entriesUnknownToPeer.values());
+		Message responseMessage = new TradePresencesMessage(safeTradePresences);
+		if (!peer.sendMessage(responseMessage)) {
+			peer.disconnect("failed to send TRADE_PRESENCES response");
+			return;
+		}
+	}
+
+	public void onTradePresencesMessage(Peer peer, Message message) {
+		TradePresencesMessage tradePresencesMessage = (TradePresencesMessage) message;
+
+		List<TradePresenceData> peersTradePresences = tradePresencesMessage.getTradePresences();
+
+		long now = NTP.getTime();
+		// Timestamps before this are too far into the past
+		long pastThreshold = now;
+		// Timestamps after this are too far into the future
+		long futureThreshold = now + PRESENCE_LIFETIME;
+
+		Map<ByteArray, Supplier<ACCT>> acctSuppliersByCodeHash = SupportedBlockchain.getAcctMap();
+
+		int newCount = 0;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			for (TradePresenceData peersTradePresence : peersTradePresences) {
+				long timestamp = peersTradePresence.getTimestamp();
+
+				// Ignore if timestamp is out of bounds
+				if (timestamp < pastThreshold || timestamp > futureThreshold) {
+					if (timestamp < pastThreshold)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too old vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+								);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is too new vs {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, pastThreshold
+						);
+
+					continue;
+				}
+
+				ByteArray pubkeyByteArray = ByteArray.of(peersTradePresence.getPublicKey());
+
+				// Ignore if we've previously verified this timestamp+publickey combo or sent timestamp is older
+				TradePresenceData existingTradeData = this.safeAllTradePresencesByPubkey.get(pubkeyByteArray);
+				if (existingTradeData != null && timestamp <= existingTradeData.getTimestamp()) {
+					if (timestamp == existingTradeData.getTimestamp())
+						LOGGER.trace("Ignoring trade presence {} from peer {} as we have verified timestamp {} before",
+								peersTradePresence.getAtAddress(), peer, timestamp
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as timestamp {} is older than latest {}",
+								peersTradePresence.getAtAddress(), peer, timestamp, existingTradeData.getTimestamp()
+						);
+
+					continue;
+				}
+
+				// Check timestamp signature
+				byte[] timestampSignature = peersTradePresence.getSignature();
+				byte[] timestampBytes = Longs.toByteArray(timestamp);
+				byte[] publicKey = peersTradePresence.getPublicKey();
+				if (!Crypto.verify(publicKey, timestampSignature, timestampBytes)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signature failed to verify",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				ATData atData = repository.getATRepository().fromATAddress(peersTradePresence.getAtAddress());
+				if (atData == null || atData.getIsFrozen() || atData.getIsFinished()) {
+					if (atData == null)
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT doesn't exist",
+								peersTradePresence.getAtAddress(), peer
+						);
+					else
+						LOGGER.trace("Ignoring trade presence {} from peer {} as AT is frozen or finished",
+								peersTradePresence.getAtAddress(), peer
+						);
+
+					continue;
+				}
+
+				ByteArray atCodeHash = new ByteArray(atData.getCodeHash());
+				Supplier<ACCT> acctSupplier = acctSuppliersByCodeHash.get(atCodeHash);
+				if (acctSupplier == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as AT isn't a known ACCT?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				CrossChainTradeData tradeData = acctSupplier.get().populateTradeData(repository, atData);
+				if (tradeData == null) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as trade data not found?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// Convert signer's public key to address form
+				String signerAddress = peersTradePresence.getTradeAddress();
+
+				// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+				if (!signerAddress.equals(tradeData.qortalCreatorTradeAddress) && !signerAddress.equals(tradeData.qortalPartnerAddress)) {
+					LOGGER.trace("Ignoring trade presence {} from peer {} as signer isn't Alice or Bob?",
+							peersTradePresence.getAtAddress(), peer
+					);
+
+					continue;
+				}
+
+				// This is new to us
+				this.allTradePresencesByPubkey.put(pubkeyByteArray, peersTradePresence);
+				++newCount;
+
+				LOGGER.trace("Added trade presence {} from peer {} with timestamp {}",
+						peersTradePresence.getAtAddress(), peer, timestamp
+				);
+
+				EventBus.INSTANCE.notify(new TradePresenceEvent(peersTradePresence));
+			}
+		} catch (DataException e) {
+			LOGGER.error("Couldn't process TRADE_PRESENCES message due to repository issue", e);
+		}
+
+		if (newCount > 0) {
+			LOGGER.debug("New trade presences: {}", newCount);
+			rebuildSafeAllTradePresences();
+		}
+	}
+
+	public void bridgePresence(long timestamp, byte[] publicKey, byte[] signature, String atAddress) {
+		long expiry = generateExpiry(timestamp);
+		ByteArray pubkeyByteArray = ByteArray.of(publicKey);
+
+		TradePresenceData fakeTradePresenceData = new TradePresenceData(expiry, publicKey, signature, atAddress);
+
+		// Only bridge if trade presence expiry timestamp is newer
+		TradePresenceData computedTradePresenceData = this.allTradePresencesByPubkey.compute(pubkeyByteArray, (k, v) ->
+				v == null || v.getTimestamp() < expiry ? fakeTradePresenceData : v
+		);
+
+		if (computedTradePresenceData == fakeTradePresenceData) {
+			LOGGER.trace("Bridged PRESENCE transaction for trade {} with timestamp {}", atAddress, expiry);
+			rebuildSafeAllTradePresences();
+
+			EventBus.INSTANCE.notify(new TradePresenceEvent(fakeTradePresenceData));
+		}
+	}
+
+	/** Decorates a CrossChainTradeData object with Alice / Bob trade-bot presence timestamp, if available. */
+	public void decorateTradeDataWithPresence(CrossChainTradeData crossChainTradeData) {
+		// Match by AT address, then check for Bob vs Alice
+		this.safeAllTradePresencesByPubkey.values().stream()
+				.filter(tradePresenceData -> tradePresenceData.getAtAddress().equals(crossChainTradeData.qortalAtAddress))
+				.forEach(tradePresenceData -> {
+					String signerAddress = tradePresenceData.getTradeAddress();
+
+					// Signer's public key (in address form) must match Bob's / Alice's trade public key (in address form)
+					if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress))
+						crossChainTradeData.creatorPresenceExpiry = tradePresenceData.getTimestamp();
+					else if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress))
+						crossChainTradeData.partnerPresenceExpiry = tradePresenceData.getTimestamp();
+				});
+	}
+
+	private long generateExpiry(long timestamp) {
+		return ((timestamp - 1) / EXPIRY_ROUNDING) * EXPIRY_ROUNDING + PRESENCE_LIFETIME;
 	}
 
 }

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -58,9 +58,14 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	 * i.e. keys with transactions but with no unspent outputs. */
 	protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
 
-	/** How many bitcoinj wallet keys to generate in each batch. */
+	/** How many wallet keys to generate in each batch. */
 	private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3;
 
+	/** How many wallet keys to generate when using bitcoinj as the data provider.
+	 * We must use a higher value here since we are unable to request multiple batches of keys.
+	 * Without this, the bitcoinj state can be missing transactions, causing errors such as "insufficient balance". */
+	private static final int WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ = 50;
+
 	/** Byte offset into raw block headers to block timestamp. */
 	private static final int TIMESTAMP_OFFSET = 4 + 32 + 32;
 
@@ -99,8 +104,9 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		try {
 			ScriptType addressType = Address.fromString(this.params, address).getOutputScriptType();
 
-			return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH;
+			return addressType == ScriptType.P2PKH || addressType == ScriptType.P2SH || addressType == ScriptType.P2WPKH;
 		} catch (AddressFormatException e) {
+			LOGGER.error(String.format("Unrecognised address format: %s", address));
 			return false;
 		}
 	}
@@ -229,6 +235,25 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		return transaction.getOutputs();
 	}
 
+	/**
+	 * Returns transactions for passed script
+	 * <p>
+	 * @throws ForeignBlockchainException if error occurs
+	 */
+	public List<TransactionHash> getAddressTransactions(byte[] scriptPubKey, boolean includeUnconfirmed) throws ForeignBlockchainException {
+		int retries = 0;
+		ForeignBlockchainException e2 = null;
+		while (retries <= 3) {
+			try {
+				return this.blockchain.getAddressTransactions(scriptPubKey, includeUnconfirmed);
+			} catch (ForeignBlockchainException e) {
+				e2 = e;
+				retries++;
+			}
+		}
+		throw(e2);
+	}
+
 	/**
 	 * Returns list of transaction hashes pertaining to passed address.
 	 * <p>
@@ -263,7 +288,17 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	 * @throws ForeignBlockchainException if error occurs
 	 */
 	public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException {
-		return this.blockchain.getTransaction(txHash);
+		int retries = 0;
+		ForeignBlockchainException e2 = null;
+		while (retries <= 3) {
+			try {
+				return this.blockchain.getTransaction(txHash);
+			} catch (ForeignBlockchainException e) {
+				e2 = e;
+				retries++;
+			}
+		}
+		throw(e2);
 	}
 
 	/**
@@ -375,7 +410,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 			Set<String> keySet = new HashSet<>();
 
 			// Set the number of consecutive empty batches required before giving up
-			final int numberOfAdditionalBatchesToSearch = 5;
+			final int numberOfAdditionalBatchesToSearch = 7;
 
 			int unusedCounter = 0;
 			int ki = 0;
@@ -391,7 +426,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 					byte[] script = ScriptBuilder.createOutputScript(address).getProgram();
 
 					// Ask for transaction history - if it's empty then key has never been used
-					List<TransactionHash> historicTransactionHashes = this.blockchain.getAddressTransactions(script, false);
+					List<TransactionHash> historicTransactionHashes = this.getAddressTransactions(script, false);
 
 					if (!historicTransactionHashes.isEmpty()) {
 						areAllKeysUnused = false;
@@ -441,19 +476,27 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		List<SimpleTransaction.Input> inputs = new ArrayList<>();
 		List<SimpleTransaction.Output> outputs = new ArrayList<>();
 
+		boolean anyOutputAddressInWallet = false;
+		boolean transactionInvolvesExternalWallet = false;
+
 		for (BitcoinyTransaction.Input input : t.inputs) {
 			try {
 				BitcoinyTransaction t2 = getTransaction(input.outputTxHash);
 				List<String> senders = t2.outputs.get(input.outputVout).addresses;
 				long inputAmount = t2.outputs.get(input.outputVout).value;
 				totalInputAmount += inputAmount;
-				for (String sender : senders) {
-					boolean addressInWallet = false;
-					if (keySet.contains(sender)) {
-						total += inputAmount;
-						addressInWallet = true;
+				if (senders != null) {
+					for (String sender : senders) {
+						boolean addressInWallet = false;
+						if (keySet.contains(sender)) {
+							total += inputAmount;
+							addressInWallet = true;
+						}
+						else {
+							transactionInvolvesExternalWallet = true;
+						}
+						inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
 					}
-					inputs.add(new SimpleTransaction.Input(sender, inputAmount, addressInWallet));
 				}
 			} catch (ForeignBlockchainException e) {
 				LOGGER.trace("Failed to retrieve transaction information {}", input.outputTxHash);
@@ -461,22 +504,39 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		}
 		if (t.outputs != null && !t.outputs.isEmpty()) {
 			for (BitcoinyTransaction.Output output : t.outputs) {
-				for (String address : output.addresses) {
-					boolean addressInWallet = false;
-					if (keySet.contains(address)) {
-						if (total > 0L) {
-							amount -= (total - output.value);
-						} else {
-							amount += output.value;
+				if (output.addresses != null) {
+					for (String address : output.addresses) {
+						boolean addressInWallet = false;
+						if (keySet.contains(address)) {
+							if (total > 0L) { // Change returned from sent amount
+								amount -= (total - output.value);
+							} else { // Amount received
+								amount += output.value;
+							}
+							addressInWallet = true;
+							anyOutputAddressInWallet = true;
 						}
-						addressInWallet = true;
+						else {
+							transactionInvolvesExternalWallet = true;
+						}
+						outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
 					}
-					outputs.add(new SimpleTransaction.Output(address, output.value, addressInWallet));
 				}
 				totalOutputAmount += output.value;
 			}
 		}
 		long fee = totalInputAmount - totalOutputAmount;
+
+		if (!anyOutputAddressInWallet) {
+			// No outputs relate to this wallet - check if any inputs did (which is signified by a positive total)
+			if (total > 0) {
+				amount = total * -1;
+			}
+		}
+		else if (!transactionInvolvesExternalWallet) {
+			// All inputs and outputs relate to this wallet, so the balance should be unaffected
+			amount = 0;
+		}
 		return new SimpleTransaction(t.txHash, t.timestamp, amount, fee, inputs, outputs);
 	}
 
@@ -569,7 +629,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 			this.keyChain = this.wallet.getActiveKeyChain();
 
 			// Set up wallet's key chain
-			this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
+			this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT_BITCOINJ);
 			this.keyChain.maybeLookAhead();
 		}
 

src/main/java/org/qortal/crosschain/ElectrumX.java

@@ -5,19 +5,7 @@ import java.math.BigDecimal;
 import java.net.InetSocketAddress;
 import java.net.Socket;
 import java.net.SocketAddress;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.EnumMap;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NoSuchElementException;
-import java.util.Random;
-import java.util.Scanner;
-import java.util.Set;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -50,6 +38,9 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	/** Error message sent by some ElectrumX servers when they don't support returning verbose transactions. */
 	private static final String VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE = "verbose transactions are currently unsupported";
 
+	private static final int RESPONSE_TIME_READINGS = 5;
+	private static final long MAX_AVG_RESPONSE_TIME = 500L; // ms
+
 	public static class Server {
 		String hostname;
 
@@ -57,6 +48,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 		ConnectionType connectionType;
 
 		int port;
+		private List<Long> responseTimes = new ArrayList<>();
 
 		public Server(String hostname, ConnectionType connectionType, int port) {
 			this.hostname = hostname;
@@ -64,6 +56,25 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 			this.port = port;
 		}
 
+		public void addResponseTime(long responseTime) {
+			while (this.responseTimes.size() > RESPONSE_TIME_READINGS) {
+				this.responseTimes.remove(0);
+			}
+			this.responseTimes.add(responseTime);
+		}
+
+		public long averageResponseTime() {
+			if (this.responseTimes.size() < RESPONSE_TIME_READINGS) {
+				// Not enough readings yet
+				return 0L;
+			}
+			OptionalDouble average = this.responseTimes.stream().mapToDouble(a -> a).average();
+			if (average.isPresent()) {
+				return Double.valueOf(average.getAsDouble()).longValue();
+			}
+			return 0L;
+		}
+
 		@Override
 		public boolean equals(Object other) {
 			if (other == this)
@@ -103,7 +114,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	private Scanner scanner;
 	private int nextId = 1;
 
-	private static final int TX_CACHE_SIZE = 200;
+	private static final int TX_CACHE_SIZE = 1000;
 	@SuppressWarnings("serial")
 	private final Map<String, BitcoinyTransaction> transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) {
 		// This method is called just after a new entry has been added
@@ -390,13 +401,36 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 				String scriptPubKey = (String) ((JSONObject) outputJson.get("scriptPubKey")).get("hex");
 				long value = BigDecimal.valueOf((Double) outputJson.get("value")).setScale(8).unscaledValue().longValue();
 
-				// address too, if present
+				// address too, if present in the "addresses" array
 				List<String> addresses = null;
 				Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses");
 				if (addressesObj instanceof JSONArray) {
 					addresses = new ArrayList<>();
-					for (Object addressObj : (JSONArray) addressesObj)
+					for (Object addressObj : (JSONArray) addressesObj) {
 						addresses.add((String) addressObj);
+					}
+				}
+
+				// some peers return a single "address" string
+				Object addressObj = ((JSONObject) outputJson.get("scriptPubKey")).get("address");
+				if (addressObj instanceof String) {
+					if (addresses == null) {
+						addresses = new ArrayList<>();
+					}
+					addresses.add((String) addressObj);
+				}
+
+				// For the purposes of Qortal we require all outputs to contain addresses
+				// Some servers omit this info, causing problems down the line with balance calculations
+				// Update: it turns out that they were just using a different key - "address" instead of "addresses"
+				// The code below can remain in place, just in case a peer returns a missing address in the future
+				if (addresses == null || addresses.isEmpty()) {
+					if (this.currentServer != null) {
+						this.uselessServers.add(this.currentServer);
+						this.closeServer(this.currentServer);
+					}
+					LOGGER.info("No output addresses returned for transaction {}", txHash);
+					throw new ForeignBlockchainException(String.format("No output addresses returned for transaction %s", txHash));
 				}
 
 				outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses));
@@ -539,6 +573,17 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 
 			while (haveConnection()) {
 				Object response = connectedRpc(method, params);
+
+				// If we have more servers and this one replied slowly, try another
+				if (!this.remainingServers.isEmpty()) {
+					long averageResponseTime = this.currentServer.averageResponseTime();
+					if (averageResponseTime > MAX_AVG_RESPONSE_TIME) {
+						LOGGER.info("Slow average response time {}ms from {} - trying another server...", averageResponseTime, this.currentServer.hostname);
+						this.closeServer();
+						break;
+					}
+				}
+
 				if (response != null)
 					return response;
 
@@ -628,6 +673,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 		String request = requestJson.toJSONString() + "\n";
 		LOGGER.trace(() -> String.format("Request: %s", request));
 
+		long startTime = System.currentTimeMillis();
 		final String response;
 
 		try {
@@ -638,7 +684,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 			return null;
 		}
 
+		long endTime = System.currentTimeMillis();
+		long responseTime = endTime-startTime;
+
 		LOGGER.trace(() -> String.format("Response: %s", response));
+		LOGGER.trace(() -> String.format("Time taken: %dms", endTime-startTime));
 
 		if (response.isEmpty())
 			// Empty response - try another server?
@@ -649,6 +699,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 			// Unexpected response - try another server?
 			return null;
 
+		// Keep track of response times
+		if (this.currentServer != null) {
+			this.currentServer.addResponseTime(responseTime);
+		}
+
 		JSONObject responseJson = (JSONObject) responseObj;
 
 		Object errorObj = responseJson.get("error");

src/main/java/org/qortal/crosschain/Litecoin.java

@@ -50,8 +50,12 @@ public class Litecoin extends Bitcoiny {
 						new Server("electrum.ltc.xurious.com", Server.ConnectionType.TCP, 50001),
 						new Server("electrum.ltc.xurious.com", Server.ConnectionType.SSL, 50002),
 						new Server("electrum-ltc.bysh.me", Server.ConnectionType.SSL, 50002),
-						new Server("ltc.rentonisk.com", Server.ConnectionType.TCP, 50001),
-						new Server("ltc.rentonisk.com", Server.ConnectionType.SSL, 50002),
+						new Server("electrum2.cipig.net", Server.ConnectionType.SSL, 20063),
+						new Server("electrum3.cipig.net", Server.ConnectionType.SSL, 20063),
+						new Server("electrum3.cipig.net", ConnectionType.TCP, 10063),
+						new Server("electrum2.cipig.net", Server.ConnectionType.TCP, 10063),
+						new Server("electrum1.cipig.net", Server.ConnectionType.SSL, 20063),
+						new Server("electrum1.cipig.net", Server.ConnectionType.TCP, 10063),
 						new Server("electrum-ltc.petrkr.net", Server.ConnectionType.SSL, 60002),
 						new Server("ltc.litepay.ch", Server.ConnectionType.SSL, 50022),
 						new Server("electrum-ltc-bysh.me", Server.ConnectionType.TCP, 50002),

src/main/java/org/qortal/data/arbitrary/ArbitraryCategoryInfo.java

@@ -0,0 +1,12 @@
+package org.qortal.data.arbitrary;
+
+public class ArbitraryCategoryInfo {
+
+    public String id;
+    public String name;
+
+    public ArbitraryCategoryInfo() {
+
+    }
+
+}

src/main/java/org/qortal/data/arbitrary/ArbitraryRelayInfo.java

@@ -9,12 +9,16 @@ public class ArbitraryRelayInfo {
     private final String signature58;
     private final Peer peer;
     private final Long timestamp;
+    private final Long requestTime;
+    private final Integer requestHops;
 
-    public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp) {
+    public ArbitraryRelayInfo(String hash58, String signature58, Peer peer, Long timestamp, Long requestTime, Integer requestHops) {
         this.hash58 = hash58;
         this.signature58 = signature58;
         this.peer = peer;
         this.timestamp = timestamp;
+        this.requestTime = requestTime;
+        this.requestHops = requestHops;
     }
 
     public boolean isValid() {
@@ -38,6 +42,14 @@ public class ArbitraryRelayInfo {
         return timestamp;
     }
 
+    public Long getRequestTime() {
+        return this.requestTime;
+    }
+
+    public Integer getRequestHops() {
+        return this.requestHops;
+    }
+
     @Override
     public String toString() {
         return String.format("%s = %s, %s, %d", this.hash58, this.signature58, this.peer, this.timestamp);

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceInfo.java

@@ -13,6 +13,7 @@ public class ArbitraryResourceInfo {
 	public Service service;
 	public String identifier;
 	public ArbitraryResourceStatus status;
+	public ArbitraryResourceMetadata metadata;
 
 	public Long size;
 

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java

@@ -0,0 +1,45 @@
+package org.qortal.data.arbitrary;
+
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Category;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.List;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ArbitraryResourceMetadata {
+
+    private String title;
+    private String description;
+    private List<String> tags;
+    private Category category;
+    private String categoryName;
+
+    public ArbitraryResourceMetadata() {
+    }
+
+    public ArbitraryResourceMetadata(String title, String description, List<String> tags, Category category) {
+        this.title = title;
+        this.description = description;
+        this.tags = tags;
+        this.category = category;
+        this.categoryName = category.getName();
+    }
+
+    public static ArbitraryResourceMetadata fromTransactionMetadata(ArbitraryDataTransactionMetadata transactionMetadata) {
+        if (transactionMetadata == null) {
+            return null;
+        }
+        String title = transactionMetadata.getTitle();
+        String description = transactionMetadata.getDescription();
+        List<String> tags = transactionMetadata.getTags();
+        Category category = transactionMetadata.getCategory();
+
+        if (title == null && description == null && tags == null && category == null) {
+            return null;
+        }
+
+        return new ArbitraryResourceMetadata(title, description, tags, category);
+    }
+}

src/main/java/org/qortal/data/crosschain/CrossChainTradeData.java

@@ -94,6 +94,12 @@ public class CrossChainTradeData {
 
 	public String acctName;
 
+	@Schema(description = "Timestamp when AT creator's trade-bot presence expires")
+	public Long creatorPresenceExpiry;
+
+	@Schema(description = "Timestamp when trade partner's trade-bot presence expires")
+	public Long partnerPresenceExpiry;
+
 	// Constructors
 
 	// Necessary for JAXB

src/main/java/org/qortal/data/network/TradePresenceData.java

@@ -0,0 +1,114 @@
+package org.qortal.data.network;
+
+import org.qortal.crypto.Crypto;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlTransient;
+import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
+import java.util.Arrays;
+
+// All properties to be converted to JSON via JAXB
+@XmlAccessorType(XmlAccessType.FIELD)
+public class TradePresenceData {
+
+	protected long timestamp;
+
+	@XmlJavaTypeAdapter(
+			type = byte[].class,
+			value = org.qortal.api.Base58TypeAdapter.class
+	)
+	protected byte[] publicKey; // Could be BOB's or ALICE's
+
+	// No need to send this via websocket / API
+	@XmlTransient
+	protected byte[] signature; // Not always present
+
+	protected String atAddress; // Not always present
+
+	// Have JAXB use getter instead
+	@XmlTransient
+	protected String tradeAddress; // Lazily instantiated
+
+	// Constructors
+
+	// necessary for JAXB serialization
+	protected TradePresenceData() {
+	}
+
+	public TradePresenceData(long timestamp, byte[] publicKey, byte[] signature, String atAddress) {
+		this.timestamp = timestamp;
+		this.publicKey = publicKey;
+		this.signature = signature;
+		this.atAddress = atAddress;
+	}
+
+	public TradePresenceData(long timestamp, byte[] publicKey) {
+		this(timestamp, publicKey, null, null);
+	}
+
+	public long getTimestamp() {
+		return this.timestamp;
+	}
+
+	public byte[] getPublicKey() {
+		return this.publicKey;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public String getAtAddress() {
+		return this.atAddress;
+	}
+
+	// Probably doesn't need synchronization
+	@XmlElement
+	public String getTradeAddress() {
+		if (tradeAddress != null)
+			return tradeAddress;
+
+		tradeAddress = Crypto.toAddress(this.publicKey);
+		return tradeAddress;
+	}
+
+	// Comparison
+
+	@Override
+	public boolean equals(Object other) {
+		if (other == this)
+			return true;
+
+		if (!(other instanceof TradePresenceData))
+			return false;
+
+		TradePresenceData otherTradePresenceData = (TradePresenceData) other;
+
+		// Very quick comparison
+		if (otherTradePresenceData.timestamp != this.timestamp)
+			return false;
+
+		if (!Arrays.equals(otherTradePresenceData.publicKey, this.publicKey))
+			return false;
+
+		if (otherTradePresenceData.atAddress != null && !otherTradePresenceData.atAddress.equals(this.atAddress))
+			return false;
+
+		if (this.atAddress != null && !this.atAddress.equals(otherTradePresenceData.atAddress))
+			return false;
+
+		if (!Arrays.equals(otherTradePresenceData.signature, this.signature))
+			return false;
+
+		return true;
+	}
+
+	@Override
+	public int hashCode() {
+		// Pretty lazy implementation
+		return (int) this.timestamp;
+	}
+
+}

src/main/java/org/qortal/network/Handshake.java

@@ -74,6 +74,12 @@ public enum Handshake {
 			peer.setPeersConnectionTimestamp(peersConnectionTimestamp);
 			peer.setPeersVersion(versionString, version);
 
+			// Ensure the peer is running at least the version specified in MIN_PEER_VERSION
+			if (peer.isAtLeastVersion(MIN_PEER_VERSION) == false) {
+				LOGGER.debug(String.format("Ignoring peer %s because it is on an old version (%s)", peer, versionString));
+				return null;
+			}
+
 			if (Settings.getInstance().getAllowConnectionsWithOlderPeerVersions() == false) {
 				// Ensure the peer is running at least the minimum version allowed for connections
 				final String minPeerVersion = Settings.getInstance().getMinPeerVersion();
@@ -258,6 +264,9 @@ public enum Handshake {
 
 	private static final long PEER_VERSION_131 = 0x0100030001L;
 
+	/** Minimum peer version that we are allowed to communicate with */
+	private static final String MIN_PEER_VERSION = "3.1.0";
+
 	private static final int POW_BUFFER_SIZE_PRE_131 = 8 * 1024 * 1024; // bytes
 	private static final int POW_DIFFICULTY_PRE_131 = 8; // leading zero bits
 	// Can always be made harder in the future...

src/main/java/org/qortal/network/Network.java

@@ -1,5 +1,6 @@
 package org.qortal.network;
 
+import com.dosse.upnp.UPnP;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.bouncycastle.crypto.params.Ed25519PrivateKeyParameters;
@@ -7,7 +8,6 @@ import org.bouncycastle.crypto.params.Ed25519PublicKeyParameters;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.Controller;
 import org.qortal.controller.arbitrary.ArbitraryDataFileListManager;
-import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.crypto.Crypto;
 import org.qortal.data.block.BlockData;
 import org.qortal.data.network.PeerData;
@@ -183,6 +183,14 @@ public class Network {
             }
         }
 
+        // Attempt to set up UPnP. All errors are ignored.
+        if (Settings.getInstance().isUPnPEnabled()) {
+            UPnP.openPortTCP(Settings.getInstance().getListenPort());
+        }
+        else {
+            UPnP.closePortTCP(Settings.getInstance().getListenPort());
+        }
+
         // Start up first networking thread
         networkEPC.start();
     }
@@ -243,12 +251,15 @@ public class Network {
     public boolean requestDataFromPeer(String peerAddressString, byte[] signature) {
         if (peerAddressString != null) {
             PeerAddress peerAddress = PeerAddress.fromString(peerAddressString);
+            PeerData peerData = null;
 
             // Reuse an existing PeerData instance if it's already in the known peers list
-            PeerData peerData = this.allKnownPeers.stream()
-                    .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
-                    .findFirst()
-                    .orElse(null);
+            synchronized (this.allKnownPeers) {
+                peerData = this.allKnownPeers.stream()
+                        .filter(knownPeerData -> knownPeerData.getAddress().equals(peerAddress))
+                        .findFirst()
+                        .orElse(null);
+            }
 
             if (peerData == null) {
                 // Not a known peer, so we need to create one
@@ -263,10 +274,13 @@ public class Network {
             }
 
             // Check if we're already connected to and handshaked with this peer
-            Peer connectedPeer = this.connectedPeers.stream()
-                    .filter(p -> p.getPeerData().getAddress().equals(peerAddress))
-                    .findFirst()
-                    .orElse(null);
+            Peer connectedPeer = null;
+            synchronized (this.connectedPeers) {
+                connectedPeer = this.connectedPeers.stream()
+                        .filter(p -> p.getPeerData().getAddress().equals(peerAddress))
+                        .findFirst()
+                        .orElse(null);
+            }
             boolean isConnected = (connectedPeer != null);
 
             boolean isHandshaked = this.getHandshakedPeers().stream()
@@ -1178,7 +1192,12 @@ public class Network {
     public void onExternalIpUpdate(String ipAddress) {
         LOGGER.info("External IP address updated to {}", ipAddress);
 
-        ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
+        //ArbitraryDataManager.getInstance().broadcastHostedSignatureList();
+    }
+
+    public String getOurExternalIpAddress() {
+        // FUTURE: replace port if UPnP is active, as it will be more accurate
+        return this.ourExternalIpAddress;
     }
 
 

src/main/java/org/qortal/network/message/ArbitraryDataFileListMessage.java

@@ -1,6 +1,8 @@
 package org.qortal.network.message;
 
 import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.PeerData;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.Transformer;
 import org.qortal.utils.Serialization;
@@ -16,22 +18,38 @@ public class ArbitraryDataFileListMessage extends Message {
 
 	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
 	private static final int HASH_LENGTH = Transformer.SHA256_LENGTH;
+	private static final int MAX_PEER_ADDRESS_LENGTH = PeerData.MAX_PEER_ADDRESS_SIZE;
 
 	private final byte[] signature;
 	private final List<byte[]> hashes;
+	private Long requestTime;
+	private Integer requestHops;
+	private String peerAddress;
+	private Boolean isRelayPossible;
 
-	public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes) {
+
+	public ArbitraryDataFileListMessage(byte[] signature, List<byte[]> hashes, Long requestTime,
+										Integer requestHops, String peerAddress, boolean isRelayPossible) {
 		super(MessageType.ARBITRARY_DATA_FILE_LIST);
 
 		this.signature = signature;
 		this.hashes = hashes;
+		this.requestTime = requestTime;
+		this.requestHops = requestHops;
+		this.peerAddress = peerAddress;
+		this.isRelayPossible = isRelayPossible;
 	}
 
-	public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes) {
+	public ArbitraryDataFileListMessage(int id, byte[] signature, List<byte[]> hashes, Long requestTime,
+										Integer requestHops, String peerAddress, boolean isRelayPossible) {
 		super(id, MessageType.ARBITRARY_DATA_FILE_LIST);
 
 		this.signature = signature;
 		this.hashes = hashes;
+		this.requestTime = requestTime;
+		this.requestHops = requestHops;
+		this.peerAddress = peerAddress;
+		this.isRelayPossible = isRelayPossible;
 	}
 
 	public List<byte[]> getHashes() {
@@ -48,9 +66,6 @@ public class ArbitraryDataFileListMessage extends Message {
 
 		int count = bytes.getInt();
 
-		if (bytes.remaining() != count * HASH_LENGTH)
-			return null;
-
 		List<byte[]> hashes = new ArrayList<>();
 		for (int i = 0; i < count; ++i) {
 
@@ -59,7 +74,26 @@ public class ArbitraryDataFileListMessage extends Message {
 			hashes.add(hash);
 		}
 
-		return new ArbitraryDataFileListMessage(id, signature, hashes);
+		Long requestTime = null;
+		Integer requestHops = null;
+		String peerAddress = null;
+		boolean isRelayPossible = true; // Legacy versions only send this message when relaying is possible
+
+		// The remaining fields are optional
+
+		if (bytes.hasRemaining()) {
+
+			requestTime = bytes.getLong();
+
+			requestHops = bytes.getInt();
+
+			peerAddress = Serialization.deserializeSizedStringV2(bytes, MAX_PEER_ADDRESS_LENGTH);
+
+			isRelayPossible = bytes.getInt() > 0;
+
+		}
+
+		return new ArbitraryDataFileListMessage(id, signature, hashes, requestTime, requestHops, peerAddress, isRelayPossible);
 	}
 
 	@Override
@@ -75,6 +109,20 @@ public class ArbitraryDataFileListMessage extends Message {
 				bytes.write(hash);
 			}
 
+			if (this.requestTime == null) { // To maintain backwards support
+				return bytes.toByteArray();
+			}
+
+			// The remaining fields are optional
+
+			bytes.write(Longs.toByteArray(this.requestTime));
+
+			bytes.write(Ints.toByteArray(this.requestHops));
+
+			Serialization.serializeSizedStringV2(bytes, this.peerAddress);
+
+			bytes.write(Ints.toByteArray(this.isRelayPossible ? 1 : 0));
+
 			return bytes.toByteArray();
 		} catch (IOException e) {
 			return null;
@@ -82,9 +130,49 @@ public class ArbitraryDataFileListMessage extends Message {
 	}
 
 	public ArbitraryDataFileListMessage cloneWithNewId(int newId) {
-		ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes);
+		ArbitraryDataFileListMessage clone = new ArbitraryDataFileListMessage(this.signature, this.hashes,
+				this.requestTime, this.requestHops, this.peerAddress, this.isRelayPossible);
 		clone.setId(newId);
 		return clone;
 	}
 
+	public void removeOptionalStats() {
+		this.requestTime = null;
+		this.requestHops = null;
+		this.peerAddress = null;
+		this.isRelayPossible = null;
+	}
+
+	public Long getRequestTime() {
+		return this.requestTime;
+	}
+
+	public void setRequestTime(Long requestTime) {
+		this.requestTime = requestTime;
+	}
+
+	public Integer getRequestHops() {
+		return this.requestHops;
+	}
+
+	public void setRequestHops(Integer requestHops) {
+		this.requestHops = requestHops;
+	}
+
+	public String getPeerAddress() {
+		return this.peerAddress;
+	}
+
+	public void setPeerAddress(String peerAddress) {
+		this.peerAddress = peerAddress;
+	}
+
+	public Boolean isRelayPossible() {
+		return this.isRelayPossible;
+	}
+
+	public void setIsRelayPossible(Boolean isRelayPossible) {
+		this.isRelayPossible = isRelayPossible;
+	}
+
 }

src/main/java/org/qortal/network/message/ArbitraryDataFileMessage.java

@@ -1,6 +1,8 @@
 package org.qortal.network.message;
 
 import com.google.common.primitives.Ints;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.repository.DataException;
 import org.qortal.transform.Transformer;
@@ -12,6 +14,8 @@ import java.nio.ByteBuffer;
 
 public class ArbitraryDataFileMessage extends Message {
 
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileMessage.class);
+
 	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
 
 	private final byte[] signature;
@@ -52,6 +56,7 @@ public class ArbitraryDataFileMessage extends Message {
 			return new ArbitraryDataFileMessage(id, signature, arbitraryDataFile);
 		}
 		catch (DataException e) {
+			LOGGER.info("Unable to process received file: {}", e.getMessage());
 			return null;
 		}
 	}

src/main/java/org/qortal/network/message/ArbitraryMetadataMessage.java

@@ -0,0 +1,95 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.repository.DataException;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+public class ArbitraryMetadataMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+
+	private final byte[] signature;
+	private final ArbitraryDataFile arbitraryMetadataFile;
+
+	public ArbitraryMetadataMessage(byte[] signature, ArbitraryDataFile arbitraryDataFile) {
+		super(MessageType.ARBITRARY_METADATA);
+
+		this.signature = signature;
+		this.arbitraryMetadataFile = arbitraryDataFile;
+	}
+
+	public ArbitraryMetadataMessage(int id, byte[] signature, ArbitraryDataFile arbitraryDataFile) {
+		super(id, MessageType.ARBITRARY_METADATA);
+
+		this.signature = signature;
+		this.arbitraryMetadataFile = arbitraryDataFile;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public ArbitraryDataFile getArbitraryMetadataFile() {
+		return this.arbitraryMetadataFile;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+		byteBuffer.get(signature);
+
+		int dataLength = byteBuffer.getInt();
+
+		if (byteBuffer.remaining() != dataLength)
+			return null;
+
+		byte[] data = new byte[dataLength];
+		byteBuffer.get(data);
+
+		try {
+			ArbitraryDataFile arbitraryMetadataFile = new ArbitraryDataFile(data, signature);
+			return new ArbitraryMetadataMessage(id, signature, arbitraryMetadataFile);
+		}
+		catch (DataException e) {
+			return null;
+		}
+	}
+
+	@Override
+	protected byte[] toData() {
+		if (this.arbitraryMetadataFile == null) {
+			return null;
+		}
+
+		byte[] data = this.arbitraryMetadataFile.getBytes();
+		if (data == null) {
+			return null;
+		}
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(signature);
+
+			bytes.write(Ints.toByteArray(data.length));
+
+			bytes.write(data);
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+	public ArbitraryMetadataMessage cloneWithNewId(int newId) {
+		ArbitraryMetadataMessage clone = new ArbitraryMetadataMessage(this.signature, this.arbitraryMetadataFile);
+		clone.setId(newId);
+		return clone;
+	}
+
+}

src/main/java/org/qortal/network/message/GetArbitraryMetadataMessage.java

@@ -0,0 +1,83 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+
+import static org.qortal.transform.Transformer.INT_LENGTH;
+import static org.qortal.transform.Transformer.LONG_LENGTH;
+
+public class GetArbitraryMetadataMessage extends Message {
+
+	private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
+
+	private final byte[] signature;
+	private final long requestTime;
+	private int requestHops;
+
+	public GetArbitraryMetadataMessage(byte[] signature, long requestTime, int requestHops) {
+		this(-1, signature, requestTime, requestHops);
+	}
+
+	private GetArbitraryMetadataMessage(int id, byte[] signature, long requestTime, int requestHops) {
+		super(id, MessageType.GET_ARBITRARY_METADATA);
+
+		this.signature = signature;
+		this.requestTime = requestTime;
+		this.requestHops = requestHops;
+	}
+
+	public byte[] getSignature() {
+		return this.signature;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		if (bytes.remaining() != SIGNATURE_LENGTH + LONG_LENGTH + INT_LENGTH)
+			return null;
+
+		byte[] signature = new byte[SIGNATURE_LENGTH];
+
+		bytes.get(signature);
+
+		long requestTime = bytes.getLong();
+
+		int requestHops = bytes.getInt();
+
+		return new GetArbitraryMetadataMessage(id, signature, requestTime, requestHops);
+	}
+
+	@Override
+	protected byte[] toData() {
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+
+			bytes.write(this.signature);
+
+			bytes.write(Longs.toByteArray(this.requestTime));
+
+			bytes.write(Ints.toByteArray(this.requestHops));
+
+			return bytes.toByteArray();
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+	public long getRequestTime() {
+		return this.requestTime;
+	}
+
+	public int getRequestHops() {
+		return this.requestHops;
+	}
+
+	public void setRequestHops(int requestHops) {
+		this.requestHops = requestHops;
+	}
+
+}

src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java

@@ -0,0 +1,117 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * For requesting online accounts info from remote peer, given our list of online accounts.
+ *
+ * Different format to V1:
+ * V1 is: number of entries, then timestamp + pubkey for each entry
+ * V2 is: groups of: number of entries, timestamp, then pubkey for each entry
+ *
+ * Also V2 only builds online accounts message once!
+ */
+public class GetOnlineAccountsV2Message extends Message {
+	private List<OnlineAccountData> onlineAccounts;
+	private byte[] cachedData;
+
+	public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
+		this(-1, onlineAccounts);
+	}
+
+	private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
+		super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
+
+		this.onlineAccounts = onlineAccounts;
+	}
+
+	public List<OnlineAccountData> getOnlineAccounts() {
+		return this.onlineAccounts;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int accountCount = bytes.getInt();
+
+		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
+
+		while (accountCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < accountCount; ++i) {
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
+			}
+
+			if (bytes.hasRemaining()) {
+				accountCount = bytes.getInt();
+			} else {
+				// we've finished
+				accountCount = 0;
+			}
+		}
+
+		return new GetOnlineAccountsV2Message(id, onlineAccounts);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no online accounts
+		if (this.onlineAccounts.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+			OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+			Long timestamp = onlineAccountData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+					OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+
+					if (onlineAccountData.getTimestamp() == timestamp)
+						bytes.write(onlineAccountData.getPublicKey());
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/network/message/GetTradePresencesMessage.java

@@ -0,0 +1,110 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.TradePresenceData;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * For requesting trade presences from remote peer, given our list of known trade presences.
+ *
+ * Groups of: number of entries, timestamp, then AT trade pubkey for each entry.
+ */
+public class GetTradePresencesMessage extends Message {
+	private List<TradePresenceData> tradePresences;
+	private byte[] cachedData;
+
+	public GetTradePresencesMessage(List<TradePresenceData> tradePresences) {
+		this(-1, tradePresences);
+	}
+
+	private GetTradePresencesMessage(int id, List<TradePresenceData> tradePresences) {
+		super(id, MessageType.GET_TRADE_PRESENCES);
+
+		this.tradePresences = tradePresences;
+	}
+
+	public List<TradePresenceData> getTradePresences() {
+		return this.tradePresences;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int groupedEntriesCount = bytes.getInt();
+
+		List<TradePresenceData> tradePresences = new ArrayList<>(groupedEntriesCount);
+
+		while (groupedEntriesCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < groupedEntriesCount; ++i) {
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				tradePresences.add(new TradePresenceData(timestamp, publicKey));
+			}
+
+			if (bytes.hasRemaining()) {
+				groupedEntriesCount = bytes.getInt();
+			} else {
+				// we've finished
+				groupedEntriesCount = 0;
+			}
+		}
+
+		return new GetTradePresencesMessage(id, tradePresences);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no trade presences
+		if (this.tradePresences.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (TradePresenceData tradePresenceData : this.tradePresences) {
+			Long timestamp = tradePresenceData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.tradePresences.size() * Transformer.PUBLIC_KEY_LENGTH;
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (TradePresenceData tradePresenceData : this.tradePresences) {
+					if (tradePresenceData.getTimestamp() == timestamp)
+						bytes.write(tradePresenceData.getPublicKey());
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/network/message/Message.java

@@ -78,6 +78,8 @@ public abstract class Message {
 
 		ONLINE_ACCOUNTS(80),
 		GET_ONLINE_ACCOUNTS(81),
+		ONLINE_ACCOUNTS_V2(82),
+		GET_ONLINE_ACCOUNTS_V2(83),
 
 		ARBITRARY_DATA(90),
 		GET_ARBITRARY_DATA(91),
@@ -91,7 +93,13 @@ public abstract class Message {
 		ARBITRARY_DATA_FILE_LIST(120),
 		GET_ARBITRARY_DATA_FILE_LIST(121),
 
-		ARBITRARY_SIGNATURES(130);
+		ARBITRARY_SIGNATURES(130),
+
+		TRADE_PRESENCES(140),
+		GET_TRADE_PRESENCES(141),
+		
+		ARBITRARY_METADATA(150),
+		GET_ARBITRARY_METADATA(151);
 
 		public final int value;
 		public final Method fromByteBufferMethod;

src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java

@@ -0,0 +1,124 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.transform.Transformer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * For sending online accounts info to remote peer.
+ *
+ * Different format to V1:
+ * V1 is: number of entries, then timestamp + sig + pubkey for each entry
+ * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
+ *
+ * Also V2 only builds online accounts message once!
+ */
+public class OnlineAccountsV2Message extends Message {
+	private List<OnlineAccountData> onlineAccounts;
+	private byte[] cachedData;
+
+	public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
+		this(-1, onlineAccounts);
+	}
+
+	private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
+		super(id, MessageType.ONLINE_ACCOUNTS_V2);
+
+		this.onlineAccounts = onlineAccounts;
+	}
+
+	public List<OnlineAccountData> getOnlineAccounts() {
+		return this.onlineAccounts;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int accountCount = bytes.getInt();
+
+		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
+
+		while (accountCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < accountCount; ++i) {
+				byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
+				bytes.get(signature);
+
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
+			}
+
+			if (bytes.hasRemaining()) {
+				accountCount = bytes.getInt();
+			} else {
+				// we've finished
+				accountCount = 0;
+			}
+		}
+
+		return new OnlineAccountsV2Message(id, onlineAccounts);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no online accounts
+		if (this.onlineAccounts.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+			OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+			Long timestamp = onlineAccountData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (int i = 0; i < this.onlineAccounts.size(); ++i) {
+					OnlineAccountData onlineAccountData = this.onlineAccounts.get(i);
+
+					if (onlineAccountData.getTimestamp() == timestamp) {
+						bytes.write(onlineAccountData.getSignature());
+
+						bytes.write(onlineAccountData.getPublicKey());
+					}
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/network/message/TradePresencesMessage.java

@@ -0,0 +1,123 @@
+package org.qortal.network.message;
+
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import org.qortal.data.network.TradePresenceData;
+import org.qortal.transform.Transformer;
+import org.qortal.utils.Base58;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.UnsupportedEncodingException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * For sending list of trade presences to remote peer.
+ *
+ * Groups of: number of entries, timestamp, then pubkey + sig + AT address for each entry.
+ */
+public class TradePresencesMessage extends Message {
+	private List<TradePresenceData> tradePresences;
+	private byte[] cachedData;
+
+	public TradePresencesMessage(List<TradePresenceData> tradePresences) {
+		this(-1, tradePresences);
+	}
+
+	private TradePresencesMessage(int id, List<TradePresenceData> tradePresences) {
+		super(id, MessageType.TRADE_PRESENCES);
+
+		this.tradePresences = tradePresences;
+	}
+
+	public List<TradePresenceData> getTradePresences() {
+		return this.tradePresences;
+	}
+
+	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
+		int groupedEntriesCount = bytes.getInt();
+
+		List<TradePresenceData> tradePresences = new ArrayList<>(groupedEntriesCount);
+
+		while (groupedEntriesCount > 0) {
+			long timestamp = bytes.getLong();
+
+			for (int i = 0; i < groupedEntriesCount; ++i) {
+				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+				bytes.get(publicKey);
+
+				byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
+				bytes.get(signature);
+
+				byte[] atAddressBytes = new byte[Transformer.ADDRESS_LENGTH];
+				bytes.get(atAddressBytes);
+				String atAddress = Base58.encode(atAddressBytes);
+
+				tradePresences.add(new TradePresenceData(timestamp, publicKey, signature, atAddress));
+			}
+
+			if (bytes.hasRemaining()) {
+				groupedEntriesCount = bytes.getInt();
+			} else {
+				// we've finished
+				groupedEntriesCount = 0;
+			}
+		}
+
+		return new TradePresencesMessage(id, tradePresences);
+	}
+
+	@Override
+	protected synchronized byte[] toData() {
+		if (this.cachedData != null)
+			return this.cachedData;
+
+		// Shortcut in case we have no trade presences
+		if (this.tradePresences.isEmpty()) {
+			this.cachedData = Ints.toByteArray(0);
+			return this.cachedData;
+		}
+
+		// How many of each timestamp
+		Map<Long, Integer> countByTimestamp = new HashMap<>();
+
+		for (TradePresenceData tradePresenceData : this.tradePresences) {
+			Long timestamp = tradePresenceData.getTimestamp();
+			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
+		}
+
+		// We should know exactly how many bytes to allocate now
+		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
+				+ this.tradePresences.size() * (Transformer.PUBLIC_KEY_LENGTH + Transformer.SIGNATURE_LENGTH + Transformer.ADDRESS_LENGTH);
+
+		try {
+			ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
+
+			for (long timestamp : countByTimestamp.keySet()) {
+				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
+
+				bytes.write(Longs.toByteArray(timestamp));
+
+				for (TradePresenceData tradePresenceData : this.tradePresences) {
+					if (tradePresenceData.getTimestamp() == timestamp) {
+						bytes.write(tradePresenceData.getPublicKey());
+
+						bytes.write(tradePresenceData.getSignature());
+
+						bytes.write(Base58.decode(tradePresenceData.getAtAddress()));
+					}
+				}
+			}
+
+			this.cachedData = bytes.toByteArray();
+			return this.cachedData;
+		} catch (IOException e) {
+			return null;
+		}
+	}
+
+}

src/main/java/org/qortal/settings/Settings.java

@@ -181,6 +181,8 @@ public class Settings {
 	private boolean isTestNet = false;
 	/** Port number for inbound peer-to-peer connections. */
 	private Integer listenPort;
+	/** Whether to attempt to open the listen port via UPnP */
+	private boolean uPnPEnabled = true;
 	/** Minimum number of peers to allow block minting / synchronization. */
 	private int minBlockchainPeers = 5;
 	/** Target number of outbound connections to peers we should make. */
@@ -195,7 +197,7 @@ public class Settings {
 	private int maxRetries = 2;
 
 	/** Minimum peer version number required in order to sync with them */
-	private String minPeerVersion = "3.0.1";
+	private String minPeerVersion = "3.1.0";
 	/** Whether to allow connections with peers below minPeerVersion
 	 * If true, we won't sync with them but they can still sync with us, and will show in the peers list
 	 * If false, sync will be blocked both ways, and they will not appear in the peers list */
@@ -243,7 +245,8 @@ public class Settings {
 	private String[] bootstrapHosts = new String[] {
 			"http://bootstrap.qortal.org",
 			"http://bootstrap2.qortal.org",
-			"http://cinfu1.crowetic.com"
+			"http://81.169.136.59",
+			"http://62.171.190.193"
 	};
 
 	// Auto-update sources
@@ -628,6 +631,10 @@ public class Settings {
 		return this.bindAddress;
 	}
 
+	public boolean isUPnPEnabled() {
+		return this.uPnPEnabled;
+	}
+
 	public int getMinBlockchainPeers() {
 		return this.minBlockchainPeers;
 	}

src/main/java/org/qortal/transaction/PresenceTransaction.java

@@ -13,6 +13,7 @@ import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.account.Account;
 import org.qortal.controller.Controller;
+import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.crosschain.ACCT;
 import org.qortal.crosschain.SupportedBlockchain;
 import org.qortal.crypto.Crypto;
@@ -191,12 +192,16 @@ public class PresenceTransaction extends Transaction {
 			CrossChainTradeData crossChainTradeData = acctSupplier.get().populateTradeData(repository, atData);
 
 			// OK if signer's public key (in address form) matches Bob's trade public key (in address form)
-			if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress))
+			if (signerAddress.equals(crossChainTradeData.qortalCreatorTradeAddress)) {
+				TradeBot.getInstance().bridgePresence(this.presenceTransactionData.getTimestamp(), this.transactionData.getCreatorPublicKey(), timestampSignature, atData.getATAddress());
 				return ValidationResult.OK;
+			}
 
 			// OK if signer's public key (in address form) matches Alice's trade public key (in address form)
-			if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress))
+			if (signerAddress.equals(crossChainTradeData.qortalPartnerAddress)) {
+				TradeBot.getInstance().bridgePresence(this.presenceTransactionData.getTimestamp(), this.transactionData.getCreatorPublicKey(), timestampSignature, atData.getATAddress());
 				return ValidationResult.OK;
+			}
 		}
 
 		return ValidationResult.AT_UNKNOWN;

src/main/java/org/qortal/transaction/RegisterNameTransaction.java

@@ -37,6 +37,15 @@ public class RegisterNameTransaction extends Transaction {
 		return Collections.emptyList();
 	}
 
+	@Override
+	public long getUnitFee(Long timestamp) {
+		// Use a higher unit fee after the fee increase timestamp
+		if (timestamp > BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp()) {
+			return BlockChain.getInstance().getNameRegistrationUnitFee();
+		}
+		return BlockChain.getInstance().getUnitFee();
+	}
+
 	// Navigation
 
 	public Account getRegistrant() {

src/main/java/org/qortal/transaction/TransferPrivsTransaction.java

@@ -58,7 +58,9 @@ public class TransferPrivsTransaction extends Transaction {
 			return ValidationResult.INVALID_ADDRESS;
 
 		// Check recipient is new account
-		if (this.repository.getAccountRepository().accountExists(this.transferPrivsTransactionData.getRecipient()))
+		AccountData recipientAccountData = this.repository.getAccountRepository().getAccount(this.transferPrivsTransactionData.getRecipient());
+		// Non-existent account data is OK, but if account data exists then reference needs to be null
+		if (recipientAccountData != null && recipientAccountData.getReference() != null)
 			return ValidationResult.ACCOUNT_ALREADY_EXISTS;
 
 		// Check sender has funds for fee

src/main/resources/blockchain.json

@@ -4,6 +4,8 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.001",
+	"nameRegistrationUnitFee": "5",
+	"nameRegistrationUnitFeeTimestamp": 1645372800000,
 	"useBrokenMD160ForAddresses": false,
 	"requireGroupForApproval": false,
 	"defaultGroupId": 0,

src/test/java/org/qortal/test/api/NamesApiTests.java

@@ -16,6 +16,8 @@ import org.qortal.test.common.ApiCommon;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
+import org.qortal.utils.NTP;
 
 public class NamesApiTests extends ApiCommon {
 
@@ -47,6 +49,7 @@ public class NamesApiTests extends ApiCommon {
 			String name = "test-name";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			assertNotNull(this.namesResource.getNamesByAddress(alice.getAddress(), null, null, null));
@@ -62,6 +65,7 @@ public class NamesApiTests extends ApiCommon {
 			String name = "test-name";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			assertNotNull(this.namesResource.getName(name));
@@ -77,6 +81,7 @@ public class NamesApiTests extends ApiCommon {
 			long price = 1_23456789L;
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Sell-name

src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java

@@ -22,6 +22,7 @@ import org.qortal.test.common.ArbitraryUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Base58;
 import org.qortal.utils.NTP;
 
@@ -153,6 +154,7 @@ public class ArbitraryDataStorageCapacityTests extends Common {
             PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
             String aliceName = "alice";
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), aliceName, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
             Path alicePath = ArbitraryUtils.generateRandomDataPath(dataLength);
             ArbitraryDataFile aliceArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(alice.getPublicKey()), alicePath, aliceName, identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize);
@@ -161,6 +163,7 @@ public class ArbitraryDataStorageCapacityTests extends Common {
             PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
             String bobName = "bob";
             transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), bobName, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, bob);
             Path bobPath = ArbitraryUtils.generateRandomDataPath(dataLength);
             ArbitraryDataFile bobArbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, Base58.encode(bob.getPublicKey()), bobPath, bobName, identifier, ArbitraryTransactionData.Method.PUT, service, bob, chunkSize);

src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java

@@ -21,7 +21,9 @@ import org.qortal.settings.Settings;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -59,25 +61,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             String name = "Test";
 
             // Register the name to Alice
-            TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create transaction
-            ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
+            ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
 
             // Add name to followed list
             assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
 
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.FOLLOWED_OR_VIEWED, Settings.getInstance().getStoragePolicy());
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
 
             // We should store but not pre-fetch data for this transaction
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
         }
     }
 
@@ -92,25 +96,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "FOLLOWED", true);
 
             // Register the name to Alice
-            TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create transaction
-            ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
+            ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
 
             // Add name to followed list
             assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
 
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
 
             // We shouldn't store or pre-fetch data for this transaction
-            assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertFalse(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
         }
     }
 
@@ -125,25 +131,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "VIEWED", true);
 
             // Register the name to Alice
-            TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create transaction
-            ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
+            ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
 
             // Add name to followed list
             assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
 
             // We should store but not pre-fetch data for this transaction
             assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
 
             // We should store but not pre-fetch data for this transaction
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
         }
     }
 
@@ -158,25 +166,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "ALL", true);
 
             // Register the name to Alice
-            TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create transaction
-            ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
+            ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
 
             // Add name to followed list
             assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
 
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
 
             // We should store and pre-fetch data for this transaction
-            assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
+            assertTrue(storageManager.canStoreData(arbitraryTransactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
         }
     }
 
@@ -191,25 +201,27 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             FieldUtils.writeField(Settings.getInstance(), "storagePolicy", "NONE", true);
 
             // Register the name to Alice
-            TransactionUtils.signAndMint(repository, new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, ""), alice);
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create transaction
-            ArbitraryTransactionData transactionData = this.createTxnWithName(repository, alice, name);
+            ArbitraryTransactionData arbitraryTransactionData = this.createTxnWithName(repository, alice, name);
 
             // Add name to followed list
             assertTrue(ResourceListManager.getInstance().addToList("followedNames", name, false));
 
             // We shouldn't store or pre-fetch data for this transaction
             assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
-            assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertFalse(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followedNames", name, false));
 
             // We shouldn't store or pre-fetch data for this transaction
-            assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
+            assertFalse(storageManager.canStoreData(arbitraryTransactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, arbitraryTransactionData));
         }
     }
 
@@ -234,7 +246,8 @@ public class ArbitraryDataStoragePolicyTests extends Common {
         Path path = Paths.get("src/test/resources/arbitrary/demo1");
 
         ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
-                repository, publicKey58, path, name, Method.PUT, Service.ARBITRARY_DATA, null);
+                repository, publicKey58, path, name, Method.PUT, Service.ARBITRARY_DATA, null,
+                null, null, null, null);
 
         txnBuilder.build();
         ArbitraryTransactionData transactionData = txnBuilder.getArbitraryTransactionData();

src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java

@@ -23,7 +23,9 @@ import org.qortal.test.common.ArbitraryUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -55,6 +57,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -149,6 +152,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -181,6 +185,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -226,6 +231,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -294,6 +300,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -343,6 +350,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -380,6 +388,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -409,6 +418,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -435,6 +445,7 @@ public class ArbitraryDataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction

src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionMetadataTests.java

@@ -9,6 +9,7 @@ import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.ArbitraryDataFile.*;
 import org.qortal.arbitrary.ArbitraryDataReader;
 import org.qortal.arbitrary.exception.MissingDataException;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.data.transaction.ArbitraryTransactionData;
@@ -20,10 +21,13 @@ import org.qortal.test.common.ArbitraryUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Base58;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.junit.Assert.*;
 
@@ -50,6 +54,7 @@ public class ArbitraryTransactionMetadataTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Create PUT transaction
@@ -73,4 +78,107 @@ public class ArbitraryTransactionMetadataTests extends Common {
         }
     }
 
+    @Test
+    public void testDescriptiveMetadata() throws DataException, IOException, MissingDataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+            String publicKey58 = Base58.encode(alice.getPublicKey());
+            String name = "TEST"; // Can be anything for this test
+            String identifier = null; // Not used for this test
+            Service service = Service.ARBITRARY_DATA;
+            int chunkSize = 100;
+            int dataLength = 900; // Actual data length will be longer due to encryption
+
+            String title = "Test title";
+            String description = "Test description";
+            List<String> tags = Arrays.asList("Test", "tag", "another tag");
+            Category category = Category.QORTAL;
+
+            // Register the name to Alice
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
+
+            // Create PUT transaction
+            Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
+            ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
+                    identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize,
+                    title, description, tags, category);
+
+            // Check the chunk count is correct
+            assertEquals(10, arbitraryDataFile.chunkCount());
+
+            // Check the metadata is correct
+            assertEquals(title, arbitraryDataFile.getMetadata().getTitle());
+            assertEquals(description, arbitraryDataFile.getMetadata().getDescription());
+            assertEquals(tags, arbitraryDataFile.getMetadata().getTags());
+            assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
+
+            // Now build the latest data state for this name
+            ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ResourceIdType.NAME, service, identifier);
+            arbitraryDataReader.loadSynchronously(true);
+            Path initialLayerPath = arbitraryDataReader.getFilePath();
+            ArbitraryDataDigest initialLayerDigest = new ArbitraryDataDigest(initialLayerPath);
+            initialLayerDigest.compute();
+
+            // Its directory hash should match the original directory hash
+            ArbitraryDataDigest path1Digest = new ArbitraryDataDigest(path1);
+            path1Digest.compute();
+            assertEquals(path1Digest.getHash58(), initialLayerDigest.getHash58());
+        }
+    }
+
+    @Test
+    public void testMetadataLengths() throws DataException, IOException, MissingDataException {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+            String publicKey58 = Base58.encode(alice.getPublicKey());
+            String name = "TEST"; // Can be anything for this test
+            String identifier = null; // Not used for this test
+            Service service = Service.ARBITRARY_DATA;
+            int chunkSize = 100;
+            int dataLength = 900; // Actual data length will be longer due to encryption
+
+            String title = "title Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat pretium";
+            String description = "description Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat pretium massa, non pulvinar mi pretium id. Ut gravida sapien vitae dui posuere tincidunt. Quisque in nibh est. Curabitur at blandit nunc, id aliquet neque. Nulla condimentum eget dolor a egestas. Vestibulum vel tincidunt ex. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Cras congue lacus in risus mattis suscipit. Quisque nisl eros, facilisis a lorem quis, vehicula bibendum.";
+            List<String> tags = Arrays.asList("tag 1", "tag 2", "tag 3 that is longer than the 20 character limit", "tag 4", "tag 5", "tag 6", "tag 7");
+            Category category = Category.CRYPTOCURRENCY;
+
+            String expectedTitle = "title Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat "; // 80 chars
+            String expectedDescription = "description Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent feugiat pretium massa, non pulvinar mi pretium id. Ut gravida sapien vitae dui posuere tincidunt. Quisque in nibh est. Curabitur at blandit nunc, id aliquet neque. Nulla condimentum eget dolor a egestas. Vestibulum vel tincidunt ex. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Cras congue lacus in risus mattis suscipit. Quisque nisl eros, facilisis a lorem quis, vehicula biben"; // 500 chars
+            List<String> expectedTags = Arrays.asList("tag 1", "tag 2", "tag 4", "tag 5", "tag 6");
+
+            // Register the name to Alice
+            RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+            TransactionUtils.signAndMint(repository, transactionData, alice);
+
+            // Create PUT transaction
+            Path path1 = ArbitraryUtils.generateRandomDataPath(dataLength);
+            ArbitraryDataFile arbitraryDataFile = ArbitraryUtils.createAndMintTxn(repository, publicKey58, path1, name,
+                    identifier, ArbitraryTransactionData.Method.PUT, service, alice, chunkSize,
+                    title, description, tags, category);
+
+            // Check the metadata is correct
+            assertEquals(expectedTitle, arbitraryDataFile.getMetadata().getTitle());
+            assertEquals(expectedDescription, arbitraryDataFile.getMetadata().getDescription());
+            assertEquals(expectedTags, arbitraryDataFile.getMetadata().getTags());
+            assertEquals(category, arbitraryDataFile.getMetadata().getCategory());
+        }
+    }
+
+    @Test
+    public void testExistingCategories() {
+        // Matching categories should be correctly located
+        assertEquals(Category.QORTAL, Category.uncategorizedValueOf("QORTAL"));
+        assertEquals(Category.TECHNOLOGY, Category.uncategorizedValueOf("TECHNOLOGY"));
+    }
+
+    @Test
+    public void testMissingCategory() {
+        // Missing or invalid categories should fall back to UNCATEGORIZED
+        assertEquals(Category.UNCATEGORIZED, Category.uncategorizedValueOf("INVALID_CATEGORY"));
+        assertEquals(Category.UNCATEGORIZED, Category.uncategorizedValueOf("Qortal")); // Case-sensitive match required
+    }
+
 }

src/test/java/org/qortal/test/arbitrary/ArbitraryTransactionTests.java

@@ -19,7 +19,9 @@ import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
 import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Base58;
+import org.qortal.utils.NTP;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -46,6 +48,7 @@ public class ArbitraryTransactionTests extends Common {
 
             // Register the name to Alice
             RegisterNameTransactionData registerNameTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "");
+            registerNameTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(registerNameTransactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, registerNameTransactionData, alice);
 
             // Set difficulty to 1

src/test/java/org/qortal/test/common/ArbitraryUtils.java

@@ -3,6 +3,7 @@ package org.qortal.test.common;
 import org.qortal.account.PrivateKeyAccount;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.repository.DataException;
@@ -16,6 +17,7 @@ import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.Random;
 
 import static org.junit.Assert.assertEquals;
@@ -23,11 +25,19 @@ import static org.junit.Assert.assertEquals;
 public class ArbitraryUtils {
 
     public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
-                                               ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
-                                               int chunkSize) throws DataException {
+                                                     ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
+                                                     int chunkSize) throws DataException {
+
+        return ArbitraryUtils.createAndMintTxn(repository, publicKey58, path, name, identifier, method, service,
+                account, chunkSize, null, null, null, null);
+    }
+
+    public static ArbitraryDataFile createAndMintTxn(Repository repository, String publicKey58, Path path, String name, String identifier,
+                                                     ArbitraryTransactionData.Method method, Service service, PrivateKeyAccount account,
+                                                     int chunkSize, String title, String description, List<String> tags, Category category) throws DataException {
 
         ArbitraryDataTransactionBuilder txnBuilder = new ArbitraryDataTransactionBuilder(
-                repository, publicKey58, path, name, method, service, identifier);
+                repository, publicKey58, path, name, method, service, identifier, title, description, tags, category);
 
         txnBuilder.setChunkSize(chunkSize);
         txnBuilder.build();

src/test/java/org/qortal/test/crosschain/TradeBotPresenceTests.java

@@ -0,0 +1,112 @@
+package org.qortal.test.crosschain;
+
+import org.junit.Test;
+import org.qortal.utils.ByteArray;
+
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Map;
+
+public class TradeBotPresenceTests {
+
+    public static final long ROUNDING = 15 * 60 * 1000L; // to nearest X mins
+    public static final long LIFETIME = 30 * 60 * 1000L; // lifetime: X mins
+    public static final long EARLY_RENEWAL_LIFETIME = 5 * 60 * 1000L; // X mins before expiry
+    public static final long CHECK_INTERVAL = 5 * 60 * 1000L; // X mins
+    public static final long MAX_TIMESTAMP = 100 * 60 * 1000L; // run tests for X mins
+
+    // We want to generate timestamps that expire 30 mins into the future, but also round to nearest X min?
+    // We want to regenerate timestamps early (e.g. 15 mins before expiry) to allow for network propagation
+
+    // We want to keep the latest timestamp for any given public key
+    // We want to reject out-of-bound timestamps from peers (>30 mins into future, not now/past)
+
+    // We want to make sure that we don't incorrectly delete an entry at 15-min and 30-min boundaries
+
+    @Test
+    public void testGeneratedExpiryTimestamps() {
+        for (long timestamp = 0; timestamp <= MAX_TIMESTAMP; timestamp += CHECK_INTERVAL) {
+            long expiry = generateExpiry(timestamp);
+
+            System.out.println(String.format("time: % 3dm, expiry: % 3dm",
+                    timestamp / 60_000L,
+                    expiry / 60_000L
+            ));
+        }
+    }
+
+    @Test
+    public void testEarlyRenewal() {
+        Long currentExpiry = null;
+
+        for (long timestamp = 0; timestamp <= MAX_TIMESTAMP; timestamp += CHECK_INTERVAL) {
+            long newExpiry = generateExpiry(timestamp);
+
+            if (currentExpiry == null || currentExpiry - timestamp <= EARLY_RENEWAL_LIFETIME) {
+                currentExpiry = newExpiry;
+            }
+
+            System.out.println(String.format("time: % 3dm, expiry: % 3dm",
+                    timestamp / 60_000L,
+                    currentExpiry / 60_000L
+            ));
+        }
+    }
+
+    @Test
+    public void testEnforceLatestTimestamp() {
+        ByteArray pubkeyByteArray = ByteArray.of("publickey".getBytes(StandardCharsets.UTF_8));
+
+        Map<ByteArray, Long> timestampsByPublicKey = new HashMap<>();
+
+        // Working backwards this time
+        for (long timestamp = MAX_TIMESTAMP; timestamp >= 0; timestamp -= CHECK_INTERVAL){
+            long newExpiry = generateExpiry(timestamp);
+
+            timestampsByPublicKey.compute(pubkeyByteArray, (k, v) ->
+               v == null || v < newExpiry ? newExpiry : v
+            );
+
+            Long currentExpiry = timestampsByPublicKey.get(pubkeyByteArray);
+
+            System.out.println(String.format("time: % 3dm, expiry: % 3dm",
+                    timestamp / 60_000L,
+                    currentExpiry / 60_000L
+            ));
+        }
+    }
+
+    @Test
+    public void testEnforcePeerExpiryBounds() {
+        System.out.println(String.format("%40s", "Our time"));
+
+        for (long ourTimestamp = 0; ourTimestamp <= MAX_TIMESTAMP; ourTimestamp += CHECK_INTERVAL) {
+            System.out.print(String.format("%s% 3dm ",
+                    ourTimestamp != 0 ? "| " : "        ",
+                    ourTimestamp / 60_000L
+            ));
+        }
+        System.out.println();
+
+        for (long peerTimestamp = 0; peerTimestamp <= MAX_TIMESTAMP; peerTimestamp += CHECK_INTERVAL) {
+            System.out.print(String.format("% 4dm ", peerTimestamp / 60_000L));
+
+            for (long ourTimestamp = 0; ourTimestamp <= MAX_TIMESTAMP; ourTimestamp += CHECK_INTERVAL) {
+                System.out.print(String.format("|    %s ",
+                        isPeerExpiryValid(ourTimestamp, peerTimestamp) ? "✔" : "✘"
+                ));
+            }
+            System.out.println();
+        }
+
+        System.out.println("Peer's expiry time");
+    }
+
+    private long generateExpiry(long timestamp) {
+        return ((timestamp - 1) / ROUNDING) * ROUNDING + LIFETIME;
+    }
+
+    private boolean isPeerExpiryValid(long nowTimestamp, long peerExpiry) {
+        return peerExpiry > nowTimestamp && peerExpiry <= LIFETIME + nowTimestamp;
+    }
+}

src/test/java/org/qortal/test/naming/BuySellTests.java

@@ -20,7 +20,9 @@ import org.qortal.test.common.BlockUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.utils.Amounts;
+import org.qortal.utils.NTP;
 
 public class BuySellTests extends Common {
 
@@ -62,6 +64,7 @@ public class BuySellTests extends Common {
 	public void testRegisterName() throws DataException {
 		// Register-name
 		RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, "{}");
+		transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 		TransactionUtils.signAndMint(repository, transactionData, alice);
 
 		String name = transactionData.getName();

src/test/java/org/qortal/test/naming/IntegrityTests.java

@@ -11,7 +11,9 @@ import org.qortal.repository.RepositoryManager;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.transaction.Transaction;
+import org.qortal.utils.NTP;
 
 import java.util.List;
 
@@ -33,6 +35,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -56,6 +59,7 @@ public class IntegrityTests extends Common {
             String data = "\uD83E\uDD73";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -82,6 +86,7 @@ public class IntegrityTests extends Common {
             String name = "initial_name";
             String data = "initial_data";
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Update the name, but keep the new name blank
@@ -116,6 +121,7 @@ public class IntegrityTests extends Common {
             String name = "initial_name";
             String data = "initial_data";
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Update the name, but keep the new name blank
@@ -143,6 +149,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -172,6 +179,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -210,6 +218,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -235,6 +244,7 @@ public class IntegrityTests extends Common {
 
             // Attempt to register the new name
             transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), newName, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             Transaction transaction = Transaction.fromData(repository, transactionData);
             transaction.sign(alice);
 
@@ -254,6 +264,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -268,6 +279,7 @@ public class IntegrityTests extends Common {
             // Attempt to register the name again
             String duplicateName = "TEST-nÁme";
             transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             Transaction transaction = Transaction.fromData(repository, transactionData);
             transaction.sign(alice);
 
@@ -287,6 +299,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -320,6 +333,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -329,6 +343,7 @@ public class IntegrityTests extends Common {
             String secondName = "new-missing-name";
             String secondNameData = "{\"data2\":true}";
             transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), secondName, secondNameData);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the second name exists and the data is correct
@@ -362,6 +377,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct
@@ -393,6 +409,7 @@ public class IntegrityTests extends Common {
             String data = "{\"age\":30}";
 
             RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+            transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
             TransactionUtils.signAndMint(repository, transactionData, alice);
 
             // Ensure the name exists and the data is correct

src/test/java/org/qortal/test/naming/MiscTests.java

@@ -3,20 +3,26 @@ package org.qortal.test.naming;
 import static org.junit.Assert.*;
 
 import java.util.List;
+import java.util.Optional;
 
+import org.apache.commons.lang3.reflect.FieldUtils;
 import org.junit.Before;
 import org.junit.Test;
 import org.qortal.account.PrivateKeyAccount;
+import org.qortal.block.BlockChain;
 import org.qortal.controller.BlockMinter;
 import org.qortal.data.transaction.*;
 import org.qortal.naming.Name;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
 import org.qortal.test.common.*;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
 import org.qortal.transaction.Transaction;
 import org.qortal.transaction.Transaction.ValidationResult;
+import org.qortal.utils.NTP;
 
 public class MiscTests extends Common {
 
@@ -34,6 +40,7 @@ public class MiscTests extends Common {
 			String data = "{\"age\":30}";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			List<String> recentNames = repository.getNameRepository().getRecentNames(0L);
@@ -53,11 +60,13 @@ public class MiscTests extends Common {
 			String data = "{\"age\":30}";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// duplicate
 			String duplicateName = "TEST-nÁme";
 			transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), duplicateName, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			Transaction transaction = Transaction.fromData(repository, transactionData);
 			transaction.sign(alice);
 
@@ -76,12 +85,14 @@ public class MiscTests extends Common {
 			String data = "{}";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// duplicate (this time registered by Bob)
 			PrivateKeyAccount bob = Common.getTestAccount(repository, "bob");
 			String duplicateName = "TEST-nÁme";
 			transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(bob), duplicateName, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			Transaction transaction = Transaction.fromData(repository, transactionData);
 			transaction.sign(alice);
 
@@ -100,12 +111,14 @@ public class MiscTests extends Common {
 			String data = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Register another name that we will later attempt to rename to first name (above)
 			String otherName = "new-name";
 			String otherData = "";
 			transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), otherName, otherData);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// we shouldn't be able to update name to existing name
@@ -129,6 +142,7 @@ public class MiscTests extends Common {
 			String data = "{\"age\":30}";
 
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			Transaction transaction = Transaction.fromData(repository, transactionData);
 			transaction.sign(alice);
 
@@ -147,6 +161,7 @@ public class MiscTests extends Common {
 			String data = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// we shouldn't be able to update name to an address
@@ -175,6 +190,7 @@ public class MiscTests extends Common {
 
 			// Register the name
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Ensure the name exists and the data is correct
@@ -201,6 +217,7 @@ public class MiscTests extends Common {
 
 			// Register the name
 			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Ensure the name exists and the data is correct
@@ -252,6 +269,7 @@ public class MiscTests extends Common {
 
 				// Register the name
 				RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+				transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 				TransactionUtils.signAndMint(repository, transactionData, alice);
 
 				// Ensure the name exists and the data is correct
@@ -283,6 +301,7 @@ public class MiscTests extends Common {
 
 				PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
 				RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+				transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 
 				// Ensure the name doesn't exist
 				assertNull(repository.getNameRepository().fromName(name));
@@ -304,4 +323,54 @@ public class MiscTests extends Common {
 		}
 	}
 
+	// test name registration fee increase
+	@Test
+	public void testRegisterNameFeeIncrease() throws DataException, IllegalAccessException {
+		try (final Repository repository = RepositoryManager.getRepository()) {
+
+			// Set nameRegistrationUnitFeeTimestamp to a time far in the future
+			long futureTimestamp = 9999999999999L; // 20 Nov 2286
+			FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", futureTimestamp, true);
+			assertEquals(futureTimestamp, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp());
+
+			// Validate unit fees pre and post timestamp
+			assertEquals(10000000, BlockChain.getInstance().getUnitFee()); // 0.1 QORT
+			assertEquals(500000000, BlockChain.getInstance().getNameRegistrationUnitFee()); // 5 QORT
+
+			// Register-name
+			PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
+			String name = "test-name";
+			String data = "{\"age\":30}";
+
+			RegisterNameTransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+			assertEquals(10000000L, transactionData.getFee().longValue());
+			TransactionUtils.signAndMint(repository, transactionData, alice);
+
+			// Set nameRegistrationUnitFeeTimestamp to a time in the past
+			Long now = NTP.getTime();
+			FieldUtils.writeField(BlockChain.getInstance(), "nameRegistrationUnitFeeTimestamp", now - 1000L, true);
+			assertEquals(now - 1000L, BlockChain.getInstance().getNameRegistrationUnitFeeTimestamp());
+
+			// Register a different name
+			// First try with the default unit fee
+			String name2 = "test-name-2";
+			transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data);
+			assertEquals(10000000L, transactionData.getFee().longValue());
+			Transaction transaction = Transaction.fromData(repository, transactionData);
+			transaction.sign(alice);
+			ValidationResult result = transaction.importAsUnconfirmed();
+			assertTrue("Transaction should be invalid", ValidationResult.INSUFFICIENT_FEE == result);
+
+			// Now try using correct fee (this is specified by the UI, via the /transaction/unitfee API endpoint)
+			transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), name2, data);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
+			assertEquals(500000000L, transactionData.getFee().longValue());
+			transaction = Transaction.fromData(repository, transactionData);
+			transaction.sign(alice);
+			result = transaction.importAsUnconfirmed();
+			assertTrue("Transaction should be valid", ValidationResult.OK == result);
+		}
+	}
+
 }

src/test/java/org/qortal/test/naming/UpdateTests.java

@@ -16,6 +16,8 @@ import org.qortal.test.common.BlockUtils;
 import org.qortal.test.common.Common;
 import org.qortal.test.common.TransactionUtils;
 import org.qortal.test.common.transaction.TestTransaction;
+import org.qortal.transaction.RegisterNameTransaction;
+import org.qortal.utils.NTP;
 
 public class UpdateTests extends Common {
 
@@ -34,6 +36,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, initialTransactionData, alice);
 
 			// Check name, reduced name, and data exist
@@ -100,6 +103,7 @@ public class UpdateTests extends Common {
 			String constantReducedName = "initia1-name";
 
 			TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, initialTransactionData, alice);
 
 			// Check initial name exists
@@ -147,6 +151,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData initialTransactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			initialTransactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(initialTransactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, initialTransactionData, alice);
 
 			// Check initial name exists
@@ -225,6 +230,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Check initial name exists
@@ -282,6 +288,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Check initial name exists
@@ -323,6 +330,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Check initial name exists
@@ -385,6 +393,7 @@ public class UpdateTests extends Common {
 			String initialData = "{\"age\":30}";
 
 			TransactionData transactionData = new RegisterNameTransactionData(TestTransaction.generateBase(alice), initialName, initialData);
+			transactionData.setFee(new RegisterNameTransaction(null, null).getUnitFee(transactionData.getTimestamp()));
 			TransactionUtils.signAndMint(repository, transactionData, alice);
 
 			// Check initial name exists

src/test/java/org/qortal/test/network/OnlineAccountsTests.java

@@ -0,0 +1,114 @@
+package org.qortal.test.network;
+
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.junit.Test;
+import org.qortal.data.network.OnlineAccountData;
+import org.qortal.network.message.*;
+import org.qortal.transform.Transformer;
+
+import java.nio.ByteBuffer;
+import java.security.Security;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class OnlineAccountsTests {
+
+    private static final Random RANDOM = new Random();
+    static {
+        // This must go before any calls to LogManager/Logger
+        System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
+
+        Security.insertProviderAt(new BouncyCastleProvider(), 0);
+        Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
+    }
+
+
+    @Test
+    public void testGetOnlineAccountsV2() throws Message.MessageException {
+        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
+
+        Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
+
+        byte[] messageBytes = messageOut.toBytes();
+        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
+
+        GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
+
+        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
+
+        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
+        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
+
+        Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
+        byte[] oldMessageBytes = oldMessageOut.toBytes();
+
+        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
+
+        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
+                onlineAccountsOut.size(),
+                numTimestamps,
+                numTimestamps != 1 ? "s" : "",
+                oldMessageBytes.length,
+                messageBytes.length));
+    }
+
+    @Test
+    public void testOnlineAccountsV2() throws Message.MessageException {
+        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
+
+        Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
+
+        byte[] messageBytes = messageOut.toBytes();
+        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
+
+        OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
+
+        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
+
+        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
+        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
+
+        Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
+        byte[] oldMessageBytes = oldMessageOut.toBytes();
+
+        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
+
+        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
+                onlineAccountsOut.size(),
+                numTimestamps,
+                numTimestamps != 1 ? "s" : "",
+                oldMessageBytes.length,
+                messageBytes.length));
+    }
+
+    private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
+        List<OnlineAccountData> onlineAccounts = new ArrayList<>();
+
+        int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
+
+        for (int t = 0; t < numTimestamps; ++t) {
+            int numAccounts = RANDOM.nextInt(3000);
+
+            for (int a = 0; a < numAccounts; ++a) {
+                byte[] sig = null;
+                if (withSignatures) {
+                    sig = new byte[Transformer.SIGNATURE_LENGTH];
+                    RANDOM.nextBytes(sig);
+                }
+
+                byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
+                RANDOM.nextBytes(pubkey);
+
+                onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
+            }
+        }
+
+        return onlineAccounts;
+    }
+
+}

src/test/resources/test-chain-v2-founder-rewards.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-leftover-reward.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-minting.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-qora-holder-extremes.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-qora-holder.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-reward-levels.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2-reward-scaling.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,

src/test/resources/test-chain-v2.json

@@ -5,6 +5,7 @@
 	"maxBlockSize": 2097152,
 	"maxBytesPerUnitFee": 1024,
 	"unitFee": "0.1",
+	"nameRegistrationUnitFee": "5",
 	"requireGroupForApproval": false,
 	"minAccountLevelToRewardShare": 5,
 	"maxRewardSharesPerMintingAccount": 20,