Speed up status rebuilding by excluding transactions that aren't hosted locally by the node.

This is a short term limit, is well above current usage levels, and can be increased substantially in future once the block minter code has been improved.

Q-Apps.md

@@ -375,11 +375,15 @@ let res = await qortalRequest({
     prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
     exactMatchNames: true, // Optional - if true, partial name matches are excluded
     default: false, // Optional - if true, only resources without identifiers are returned
+    mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
+    minLevel: 1, // Optional - whether to filter results by minimum account level
     includeStatus: false, // Optional - will take time to respond, so only request if necessary
     includeMetadata: false, // Optional - will take time to respond, so only request if necessary
     nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
     followedOnly: false, // Optional - include followed names only
     excludeBlocked: false, // Optional - exclude blocked content
+    // before: 1683546000000, // Optional - limit to resources created before timestamp
+    // after: 1683546000000, // Optional - limit to resources created after timestamp
     limit: 100,
     offset: 0,
     reverse: true
@@ -395,12 +399,16 @@ let res = await qortalRequest({
     identifier: "search query goes here", // Optional - searches only the "identifier" field
     names: ["QortalDemo", "crowetic", "AlphaX"], // Optional - searches only the "name" field for any of the supplied names
     prefix: false, // Optional - if true, only the beginning of fields are matched in all of the above filters
+    exactMatchNames: true, // Optional - if true, partial name matches are excluded
     default: false, // Optional - if true, only resources without identifiers are returned
+    mode: "LATEST", // Optional - whether to return all resources or just the latest for a name/service combination. Possible values: ALL,LATEST. Default: LATEST
     includeStatus: false, // Optional - will take time to respond, so only request if necessary
     includeMetadata: false, // Optional - will take time to respond, so only request if necessary
     nameListFilter: "QApp1234Subscriptions", // Optional - will only return results if they are from a name included in supplied list
     followedOnly: false, // Optional - include followed names only
     excludeBlocked: false, // Optional - exclude blocked content
+    // before: 1683546000000, // Optional - limit to resources created before timestamp
+    // after: 1683546000000, // Optional - limit to resources created after timestamp
     limit: 100,
     offset: 0,
     reverse: true

pom.xml

@@ -3,7 +3,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>org.qortal</groupId>
 	<artifactId>qortal</artifactId>
-	<version>4.2.0</version>
+	<version>4.2.2</version>
 	<packaging>jar</packaging>
 	<properties>
 		<skipTests>true</skipTests>

src/main/java/org/qortal/api/SearchMode.java

@@ -0,0 +1,6 @@
+package org.qortal.api;
+
+public enum SearchMode {
+    LATEST,
+    ALL;
+}

src/main/java/org/qortal/api/gateway/resource/GatewayResource.java

@@ -3,6 +3,8 @@ package org.qortal.api.gateway.resource;
 import io.swagger.v3.oas.annotations.security.SecurityRequirement;
 import io.swagger.v3.oas.annotations.tags.Tag;
 import org.apache.commons.lang3.StringUtils;
+import org.qortal.api.ApiError;
+import org.qortal.api.ApiExceptionFactory;
 import org.qortal.api.Security;
 import org.qortal.arbitrary.ArbitraryDataFile;
 import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
@@ -11,6 +13,9 @@ import org.qortal.arbitrary.ArbitraryDataRenderer;
 import org.qortal.arbitrary.ArbitraryDataResource;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
 
 import javax.servlet.ServletContext;
 import javax.servlet.http.HttpServletRequest;
@@ -31,36 +36,12 @@ public class GatewayResource {
     @Context HttpServletResponse response;
     @Context ServletContext context;
 
-    /**
-     * We need to allow resource status checking (and building) via the gateway, as the node's API port
-     * may not be forwarded and will almost certainly not be authenticated. Since gateways allow for
-     * all resources to be loaded except those that are blocked, there is no need for authentication.
-     */
-    @GET
-    @Path("/arbitrary/resource/status/{service}/{name}")
-    public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
-                                                             @PathParam("name") String name,
-                                                             @QueryParam("build") Boolean build) {
-
-        return this.getStatus(service, name, null, build);
-    }
-
-    @GET
-    @Path("/arbitrary/resource/status/{service}/{name}/{identifier}")
-    public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
-                                                      @PathParam("name") String name,
-                                                      @PathParam("identifier") String identifier,
-                                                      @QueryParam("build") Boolean build) {
-
-        return this.getStatus(service, name, identifier, build);
-    }
-
     private ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
 
         // If "build=true" has been specified in the query string, build the resource before returning its status
         if (build != null && build == true) {
-            ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
             try {
+                ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, null);
                 if (!reader.isBuilding()) {
                     reader.loadSynchronously(false);
                 }
@@ -69,8 +50,13 @@ public class GatewayResource {
             }
         }
 
-        ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
-        return resource.getStatus(false);
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
+            return resource.getStatus(repository);
+
+        } catch (DataException e) {
+            throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+        }
     }
 
 

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -45,6 +45,7 @@ import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
+import org.qortal.controller.arbitrary.ArbitraryDataCacheManager;
 import org.qortal.controller.arbitrary.ArbitraryDataRenderManager;
 import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.controller.arbitrary.ArbitraryMetadataManager;
@@ -86,12 +87,12 @@ public class ArbitraryResource {
 					"- If default is set to true, only resources without identifiers will be returned.",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> getResources(
+	public List<ArbitraryResourceData> getResources(
 			@QueryParam("service") Service service,
 			@QueryParam("name") String name,
 			@QueryParam("identifier") String identifier,
@@ -133,20 +134,14 @@ public class ArbitraryResource {
 				}
 			}
 
-			List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
-					.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
+			List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
+					.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked,
+							includeMetadata, includeStatus, limit, offset, reverse);
 
 			if (resources == null) {
 				return new ArrayList<>();
 			}
 
-			if (includeStatus != null && includeStatus) {
-				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
-			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
-
 			return resources;
 
 		} catch (DataException e) {
@@ -161,24 +156,30 @@ public class ArbitraryResource {
 					"If default is set to true, only resources without identifiers will be returned.",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> searchResources(
+	public List<ArbitraryResourceData> searchResources(
 			@QueryParam("service") Service service,
-			@Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query,
+			@Parameter(description = "Query (searches name, identifier, title and description fields)") @QueryParam("query") String query,
 			@Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier,
 			@Parameter(description = "Name (searches name field only)") @QueryParam("name") List<String> names,
+			@Parameter(description = "Title (searches title metadata field only)") @QueryParam("title") String title,
+			@Parameter(description = "Description (searches description metadata field only)") @QueryParam("description") String description,
 			@Parameter(description = "Prefix only (if true, only the beginning of fields are matched)") @QueryParam("prefix") Boolean prefixOnly,
 			@Parameter(description = "Exact match names only (if true, partial name matches are excluded)") @QueryParam("exactmatchnames") Boolean exactMatchNamesOnly,
 			@Parameter(description = "Default resources (without identifiers) only") @QueryParam("default") Boolean defaultResource,
+			@Parameter(description = "Search mode") @QueryParam("mode") SearchMode mode,
+			@Parameter(description = "Min level") @QueryParam("minlevel") Integer minLevel,
 			@Parameter(description = "Filter names by list (exact matches only)") @QueryParam("namefilter") String nameListFilter,
 			@Parameter(description = "Include followed names only") @QueryParam("followedonly") Boolean followedOnly,
 			@Parameter(description = "Exclude blocked content") @QueryParam("excludeblocked") Boolean excludeBlocked,
 			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
 			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
+			@Parameter(description = "Creation date before timestamp") @QueryParam("before") Long before,
+			@Parameter(description = "Creation date after timestamp") @QueryParam("after") Long after,
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@Parameter(ref = "reverse") @QueryParam("reverse") Boolean reverse) {
@@ -206,20 +207,15 @@ public class ArbitraryResource {
 				names = null;
 			}
 
-			List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
-					.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
+			List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
+					.searchArbitraryResources(service, query, identifier, names, title, description, usePrefixOnly,
+							exactMatchNames, defaultRes, mode, minLevel, followedOnly, excludeBlocked, includeMetadata, includeStatus,
+							before, after, limit, offset, reverse);
 
 			if (resources == null) {
 				return new ArrayList<>();
 			}
 
-			if (includeStatus != null && includeStatus) {
-				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
-			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
-
 			return resources;
 
 		} catch (DataException e) {
@@ -238,16 +234,14 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
-	public ArbitraryResourceStatus getDefaultResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-															@PathParam("service") Service service,
+	public ArbitraryResourceStatus getDefaultResourceStatus(@PathParam("service") Service service,
 															@PathParam("name") String name,
 															@QueryParam("build") Boolean build) {
 
 		if (!Settings.getInstance().isQDNAuthBypassEnabled())
-			Security.requirePriorAuthorizationOrApiKey(request, name, service, null, apiKey);
+			Security.requirePriorAuthorizationOrApiKey(request, name, service, null, null);
 
-		return ArbitraryTransactionUtils.getStatus(service, name, null, build);
+		return ArbitraryTransactionUtils.getStatus(service, name, null, build, true);
 	}
 
 	@GET
@@ -261,14 +255,12 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
-	public FileProperties getResourceProperties(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-											  @PathParam("service") Service service,
-											  @PathParam("name") String name,
-											  @PathParam("identifier") String identifier) {
+	public FileProperties getResourceProperties(@PathParam("service") Service service,
+											    @PathParam("name") String name,
+											    @PathParam("identifier") String identifier) {
 
 		if (!Settings.getInstance().isQDNAuthBypassEnabled())
-			Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey);
+			Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null);
 
 		return this.getFileProperties(service, name, identifier);
 	}
@@ -284,17 +276,15 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
-	public ArbitraryResourceStatus getResourceStatus(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-													 @PathParam("service") Service service,
+	public ArbitraryResourceStatus getResourceStatus(@PathParam("service") Service service,
 													 @PathParam("name") String name,
 													 @PathParam("identifier") String identifier,
 													 @QueryParam("build") Boolean build) {
 
 		if (!Settings.getInstance().isQDNAuthBypassEnabled())
-			Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, apiKey);
+			Security.requirePriorAuthorizationOrApiKey(request, name, service, identifier, null);
 
-		return ArbitraryTransactionUtils.getStatus(service, name, identifier, build);
+		return ArbitraryTransactionUtils.getStatus(service, name, identifier, build, true);
 	}
 
 
@@ -479,21 +469,19 @@ public class ArbitraryResource {
 			summary = "List arbitrary resources hosted by this node",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> getHostedResources(
+	public List<ArbitraryResourceData> getHostedResources(
 			@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
-			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@QueryParam("query") String query) {
 		Security.checkApiCallAllowed(request);
 
-		List<ArbitraryResourceInfo> resources = new ArrayList<>();
+		List<ArbitraryResourceData> resources = new ArrayList<>();
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			
@@ -509,22 +497,15 @@ public class ArbitraryResource {
 				if (transactionData.getService() == null) {
 					continue;
 				}
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
-				arbitraryResourceInfo.name = transactionData.getName();
-				arbitraryResourceInfo.service = transactionData.getService();
-				arbitraryResourceInfo.identifier = transactionData.getIdentifier();
-				if (!resources.contains(arbitraryResourceInfo)) {
-					resources.add(arbitraryResourceInfo);
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+				arbitraryResourceData.name = transactionData.getName();
+				arbitraryResourceData.service = transactionData.getService();
+				arbitraryResourceData.identifier = transactionData.getIdentifier();
+				if (!resources.contains(arbitraryResourceData)) {
+					resources.add(arbitraryResourceData);
 				}
 			}
 
-			if (includeStatus != null && includeStatus) {
-				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
-			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
-
 			return resources;
 
 		} catch (DataException e) {
@@ -551,8 +532,14 @@ public class ArbitraryResource {
 								  @PathParam("identifier") String identifier) {
 
 		Security.checkApiCallAllowed(request);
-		ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
-		return resource.delete(false);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
+			return resource.delete(repository, false);
+
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
+		}
 	}
 
 	@POST
@@ -644,9 +631,7 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
-	public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-								   @PathParam("service") Service service,
+	public HttpServletResponse get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @QueryParam("filepath") String filepath,
 								   @QueryParam("encoding") String encoding,
@@ -679,9 +664,7 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
-	public HttpServletResponse get(@HeaderParam(Security.API_KEY_HEADER) String apiKey,
-								   @PathParam("service") Service service,
+	public HttpServletResponse get(@PathParam("service") Service service,
 								   @PathParam("name") String name,
 								   @PathParam("identifier") String identifier,
 								   @QueryParam("filepath") String filepath,
@@ -692,7 +675,7 @@ public class ArbitraryResource {
 
 		// Authentication can be bypassed in the settings, for those running public QDN nodes
 		if (!Settings.getInstance().isQDNAuthBypassEnabled()) {
-			Security.checkApiCallAllowed(request, apiKey);
+			Security.checkApiCallAllowed(request, null);
 		}
 
 		return this.download(service, name, identifier, filepath, encoding, rebuild, async, attempts);
@@ -717,7 +700,6 @@ public class ArbitraryResource {
 					)
 			}
 	)
-	@SecurityRequirement(name = "apiKey")
 	public ArbitraryResourceMetadata getMetadata(@PathParam("service") Service service,
 							  					 @PathParam("name") String name,
 							  					 @PathParam("identifier") String identifier) {
@@ -1127,6 +1109,36 @@ public class ArbitraryResource {
 	}
 
 
+	@POST
+	@Path("/resources/cache/rebuild")
+	@Operation(
+			summary = "Rebuild arbitrary resources cache from transactions",
+			responses = {
+					@ApiResponse(
+							description = "true on success",
+							content = @Content(
+									mediaType = MediaType.TEXT_PLAIN,
+									schema = @Schema(
+											type = "boolean"
+									)
+							)
+					)
+			}
+	)
+	@SecurityRequirement(name = "apiKey")
+	public String rebuildCache(@HeaderParam(Security.API_KEY_HEADER) String apiKey) {
+		Security.checkApiCallAllowed(request);
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, true);
+
+			return "true";
+		} catch (DataException e) {
+			throw ApiExceptionFactory.INSTANCE.createCustomException(request, ApiError.REPOSITORY_ISSUE, e.getMessage());
+		}
+	}
+
+
 	// Shared methods
 
 	private String preview(String directoryPath, Service service) {
@@ -1275,8 +1287,8 @@ public class ArbitraryResource {
 
 	private HttpServletResponse download(Service service, String name, String identifier, String filepath, String encoding, boolean rebuild, boolean async, Integer maxAttempts) {
 
-		ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
 		try {
+			ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
 
 			int attempts = 0;
 			if (maxAttempts == null) {
@@ -1382,8 +1394,8 @@ public class ArbitraryResource {
 	}
 
 	private FileProperties getFileProperties(Service service, String name, String identifier) {
-		ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
 		try {
+			ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
 			arbitraryDataReader.loadSynchronously(false);
 			java.nio.file.Path outputPath = arbitraryDataReader.getFilePath();
 			if (outputPath == null) {

src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@@ -4,6 +4,7 @@ import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.arbitrary.ArbitraryDataFile.*;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.repository.DataException;
+import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
@@ -51,6 +52,9 @@ public class ArbitraryDataBuildQueueItem extends ArbitraryDataResource {
             arbitraryDataReader.loadSynchronously(true);
         } finally {
             this.buildEndTimestamp = NTP.getTime();
+
+            // Update status after build
+            ArbitraryTransactionUtils.getStatus(service, resourceId, identifier, false, true);
         }
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -66,7 +66,7 @@ public class ArbitraryDataReader {
     // TODO: all builds could be handled by the build queue (even synchronous ones), to avoid the need for this
     private static Map<String, Long> inProgress = Collections.synchronizedMap(new HashMap<>());
 
-    public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
+    public ArbitraryDataReader(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) throws DataException {
         // Ensure names are always lowercase
         if (resourceIdType == ResourceIdType.NAME) {
             resourceId = resourceId.toLowerCase();
@@ -90,11 +90,16 @@ public class ArbitraryDataReader {
         this.canRequestMissingFiles = true;
     }
 
-    private Path buildWorkingPath() {
+    private Path buildWorkingPath() throws DataException {
         // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
         String baseDir = Settings.getInstance().getTempDataPath();
         String identifier = this.identifier != null ?  this.identifier : "default";
-        return Paths.get(baseDir, "reader", this.resourceIdType.toString(), this.resourceId, this.service.toString(), identifier);
+
+        try {
+            return Paths.get(baseDir, "reader", this.resourceIdType.toString(), StringUtils.sanitizeString(this.resourceId), this.service.toString(), StringUtils.sanitizeString(identifier));
+        } catch (InvalidPathException e) {
+            throw new DataException(String.format("Invalid path: %s", e.getMessage()));
+        }
     }
 
     public boolean isCachedDataAvailable() {
@@ -240,7 +245,7 @@ public class ArbitraryDataReader {
         try {
             Files.createDirectories(this.workingPath);
         } catch (IOException e) {
-            throw new DataException("Unable to create temp directory");
+            throw new DataException(String.format("Unable to create temp directory %s: %s", this.workingPath, e.getMessage()));
         }
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataRenderer.java

@@ -76,9 +76,11 @@ public class ArbitraryDataRenderer {
             return ArbitraryDataRenderer.getResponse(response, 500, "QDN is disabled in settings");
         }
 
-        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier);
-        arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
+        ArbitraryDataReader arbitraryDataReader;
         try {
+            arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service, identifier);
+            arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
+
             if (!arbitraryDataReader.isCachedDataAvailable()) {
                 // If async is requested, show a loading screen whilst build is in progress
                 if (async) {

src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java

@@ -9,6 +9,7 @@ import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.repository.DataException;
@@ -57,15 +58,39 @@ public class ArbitraryDataResource {
         this.identifier = identifier;
     }
 
-    public ArbitraryResourceStatus getStatus(boolean quick) {
-        // Calculate the chunk counts
-        // Avoid this for "quick" statuses, to speed things up
-        if (!quick) {
-            this.calculateChunkCounts();
+    public ArbitraryResourceStatus getStatusAndUpdateCache(boolean updateCache) {
+        ArbitraryResourceStatus arbitraryResourceStatus = null;
 
-            if (!this.exists) {
-                return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount);
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            arbitraryResourceStatus = this.getStatus(repository);
+
+            if (updateCache) {
+                // Update cache if possible
+                ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null;
+                ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData(this.service, this.resourceId, this.identifier);
+                repository.discardChanges();
+                repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
+                repository.saveChanges();
             }
+        } catch (DataException e) {
+            LOGGER.info("Unable to update status cache for resource {}: {}", this.toString(), e.getMessage());
+        }
+
+        return arbitraryResourceStatus;
+    }
+
+    /**
+     * Get current status of resource
+     *
+     * @param repository
+     * @return the resource's status
+     */
+    public ArbitraryResourceStatus getStatus(Repository repository) {
+        // Calculate the chunk counts
+        this.calculateChunkCounts(repository);
+
+        if (!this.exists) {
+            return new ArbitraryResourceStatus(Status.NOT_PUBLISHED, this.localChunkCount, this.totalChunkCount);
         }
 
         if (resourceIdType != ResourceIdType.NAME) {
@@ -86,18 +111,23 @@ public class ArbitraryDataResource {
         }
 
         // Firstly check the cache to see if it's already built
-        ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
-                resourceId, resourceIdType, service, identifier);
-        if (arbitraryDataReader.isCachedDataAvailable()) {
-            return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
+        try {
+            ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
+                    resourceId, resourceIdType, service, identifier);
+            if (arbitraryDataReader.isCachedDataAvailable()) {
+                return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
+            }
+        } catch (DataException e) {
+            // Assume no usable data
+            return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
         }
 
         // Check if we have all data locally for this resource
-        if (!this.allFilesDownloaded()) {
-            if (this.isDownloading()) {
+        if (!this.allFilesDownloaded(repository)) {
+            if (this.isDownloading(repository)) {
                 return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
             }
-            else if (this.isDataPotentiallyAvailable()) {
+            else if (this.isDataPotentiallyAvailable(repository)) {
                 return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
             }
             return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
@@ -139,9 +169,9 @@ public class ArbitraryDataResource {
         return null;
     }
 
-    public boolean delete(boolean deleteMetadata) {
+    public boolean delete(Repository repository, boolean deleteMetadata) {
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 return false;
             }
@@ -190,7 +220,7 @@ public class ArbitraryDataResource {
         }
     }
 
-    private boolean allFilesDownloaded() {
+    private boolean allFilesDownloaded(Repository repository) {
         // Use chunk counts to speed things up if we can
         if (this.localChunkCount != null && this.totalChunkCount != null &&
                 this.localChunkCount >= this.totalChunkCount) {
@@ -198,7 +228,7 @@ public class ArbitraryDataResource {
         }
 
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 return false;
             }
@@ -218,9 +248,14 @@ public class ArbitraryDataResource {
         }
     }
 
-    private void calculateChunkCounts() {
+    /**
+     * Calculate chunk counts of a resource
+     *
+     * @param repository optional - a new instance will be created if null
+     */
+    private void calculateChunkCounts(Repository repository) {
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 this.exists = false;
                 this.localChunkCount = 0;
@@ -245,9 +280,9 @@ public class ArbitraryDataResource {
         } catch (DataException e) {}
     }
 
-    private boolean isRateLimited() {
+    private boolean isRateLimited(Repository repository) {
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 return true;
             }
@@ -271,9 +306,9 @@ public class ArbitraryDataResource {
      * This is only used to give an indication to the user of progress
      * @return - whether data might be available on the network
      */
-    private boolean isDataPotentiallyAvailable() {
+    private boolean isDataPotentiallyAvailable(Repository repository) {
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 return false;
             }
@@ -306,9 +341,9 @@ public class ArbitraryDataResource {
      * This is only used to give an indication to the user of progress
      * @return - whether we are trying to download the resource
      */
-    private boolean isDownloading() {
+    private boolean isDownloading(Repository repository) {
         try {
-            this.fetchTransactions();
+            this.fetchTransactions(repository);
             if (this.transactions == null) {
                 return false;
             }
@@ -339,15 +374,19 @@ public class ArbitraryDataResource {
     }
 
 
-
-    private void fetchTransactions() throws DataException {
+    /**
+     * Fetch relevant arbitrary transactions for resource
+     *
+     * @param repository
+     * @throws DataException
+     */
+    private void fetchTransactions(Repository repository) throws DataException {
         if (this.transactions != null && !this.transactions.isEmpty()) {
             // Already fetched
             return;
         }
 
-        try (final Repository repository = RepositoryManager.getRepository()) {
-
+        try {
             // Get the most recent PUT
             ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
                     .getLatestTransaction(this.resourceId, this.service, ArbitraryTransactionData.Method.PUT, this.identifier);

src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java

@@ -117,8 +117,9 @@ public class ArbitraryDataTransactionBuilder {
     }
 
     private Method determineMethodAutomatically() throws DataException {
-        ArbitraryDataReader reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
+        ArbitraryDataReader reader;
         try {
+            reader = new ArbitraryDataReader(this.name, ResourceIdType.NAME, this.service, this.identifier);
             reader.loadSynchronously(true);
         } catch (Exception e) {
             // Catch all exceptions if the existing resource cannot be loaded first time

src/main/java/org/qortal/arbitrary/misc/Category.java

@@ -67,9 +67,12 @@ public enum Category {
     /**
      * Same as valueOf() but with fallback to UNCATEGORIZED if there's no match
      * @param name
-     * @return a Category (using UNCATEGORIZED if no match found)
+     * @return a Category (using UNCATEGORIZED if no match found), or null if null name passed
      */
     public static Category uncategorizedValueOf(String name) {
+        if (name == null) {
+            return null;
+        }
         try {
             return Category.valueOf(name);
         }

src/main/java/org/qortal/controller/BlockMinter.java

@@ -380,9 +380,13 @@ public class BlockMinter extends Thread {
 						parentSignatureForLastLowWeightBlock = null;
 						timeOfLastLowWeightBlock = null;
 
+						Long unconfirmedStartTime = NTP.getTime();
+
 						// Add unconfirmed transactions
 						addUnconfirmedTransactions(repository, newBlock);
 
+						LOGGER.info(String.format("Adding %d unconfirmed transactions took %d ms", newBlock.getTransactions().size(), (NTP.getTime()-unconfirmedStartTime)));
+
 						// Sign to create block's signature
 						newBlock.sign();
 
@@ -484,6 +488,9 @@ public class BlockMinter extends Thread {
 		// Sign to create block's signature, needed by Block.isValid()
 		newBlock.sign();
 
+		// User-defined limit per block
+		int limit = Settings.getInstance().getMaxTransactionsPerBlock();
+
 		// Attempt to add transactions until block is full, or we run out
 		// If a transaction makes the block invalid then skip it and it'll either expire or be in next block.
 		for (TransactionData transactionData : unconfirmedTransactions) {
@@ -496,6 +503,12 @@ public class BlockMinter extends Thread {
 				LOGGER.debug(() -> String.format("Skipping invalid transaction %s during block minting", Base58.encode(transactionData.getSignature())));
 				newBlock.deleteTransaction(transactionData);
 			}
+
+			// User-defined limit per block
+			List<Transaction> transactions = newBlock.getTransactions();
+			if (transactions != null && transactions.size() >= limit) {
+				break;
+			}
 		}
 	}
 

src/main/java/org/qortal/controller/Controller.java

@@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.network.PeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ChatTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.event.Event;
@@ -403,6 +404,7 @@ public class Controller extends Thread {
 
 			try (final Repository repository = RepositoryManager.getRepository()) {
 				RepositoryManager.rebuildTransactionSequences(repository);
+				ArbitraryDataCacheManager.getInstance().buildArbitraryResourcesCache(repository, false);
 			}
 		} catch (DataException e) {
 			// If exception has no cause or message then repository is in use by some other process.
@@ -448,6 +450,13 @@ public class Controller extends Thread {
 				Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
 				return;
 			}
+			if (ArbitraryDataCacheManager.getInstance().needsArbitraryResourcesCacheRebuild(repository)) {
+				// Don't allow the node to start if arbitrary resources cache hasn't been built yet
+				// This is needed to handle a case when bootstrapping
+				LOGGER.error("Database upgrade needed. Please restart the core to complete the upgrade process.");
+				Gui.getInstance().fatalError("Database upgrade needed", "Please restart the core to complete the upgrade process.");
+				return;
+			}
 		} catch (DataException e) {
 			LOGGER.error("Error checking transaction sequences in repository", e);
 			return;
@@ -496,6 +505,7 @@ public class Controller extends Thread {
 		LOGGER.info("Starting arbitrary-transaction controllers");
 		ArbitraryDataManager.getInstance().start();
 		ArbitraryDataFileManager.getInstance().start();
+		ArbitraryDataCacheManager.getInstance().start();
 		ArbitraryDataBuildManager.getInstance().start();
 		ArbitraryDataCleanupManager.getInstance().start();
 		ArbitraryDataStorageManager.getInstance().start();
@@ -907,6 +917,7 @@ public class Controller extends Thread {
 				if (now >= transaction.getDeadline()) {
 					LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
 					repository.getTransactionRepository().delete(transactionData);
+					this.onExpiredTransaction(transactionData);
 					deletedCount++;
 				}
 			}
@@ -949,6 +960,7 @@ public class Controller extends Thread {
 				LOGGER.info("Shutting down arbitrary-transaction controllers");
 				ArbitraryDataManager.getInstance().shutdown();
 				ArbitraryDataFileManager.getInstance().shutdown();
+				ArbitraryDataCacheManager.getInstance().shutdown();
 				ArbitraryDataBuildManager.getInstance().shutdown();
 				ArbitraryDataCleanupManager.getInstance().shutdown();
 				ArbitraryDataStorageManager.getInstance().shutdown();
@@ -1219,6 +1231,21 @@ public class Controller extends Thread {
 		});
 	}
 
+	/**
+	 * Callback for when we've deleted an expired, unconfirmed transaction.
+	 * <p>
+	 * @implSpec performs actions in a new thread
+	 */
+	public void onExpiredTransaction(TransactionData transactionData) {
+		this.callbackExecutor.execute(() -> {
+
+			// If this is an ARBITRARY transaction, we may need to update the cache
+			if (transactionData.getType() == TransactionType.ARBITRARY) {
+				ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData);
+			}
+		});
+	}
+
 	public void onPeerHandshakeCompleted(Peer peer) {
 		// Only send if outbound
 		if (peer.isOutbound()) {
@@ -1278,13 +1305,6 @@ public class Controller extends Thread {
 				TransactionImporter.getInstance().onNetworkTransactionSignaturesMessage(peer, message);
 				break;
 
-			case GET_ONLINE_ACCOUNTS:
-			case ONLINE_ACCOUNTS:
-			case GET_ONLINE_ACCOUNTS_V2:
-			case ONLINE_ACCOUNTS_V2:
-				// No longer supported - to be eventually removed
-				break;
-
 			case GET_ONLINE_ACCOUNTS_V3:
 				OnlineAccountsManager.getInstance().onNetworkGetOnlineAccountsV3Message(peer, message);
 				break;

src/main/java/org/qortal/controller/OnlineAccountsManager.java

@@ -414,7 +414,7 @@ public class OnlineAccountsManager {
         boolean isSuperiorEntry = isOnlineAccountsDataSuperior(onlineAccountData);
         if (isSuperiorEntry)
             // Remove existing inferior entry so it can be re-added below (it's likely the existing copy is missing a nonce value)
-            onlineAccounts.remove(onlineAccountData);
+            onlineAccounts.removeIf(a -> Objects.equals(a.getPublicKey(), onlineAccountData.getPublicKey()));
 
         boolean isNewEntry = onlineAccounts.add(onlineAccountData);
 
@@ -743,14 +743,8 @@ public class OnlineAccountsManager {
         if (onlineAccounts == null)
             onlineAccounts = this.latestBlocksOnlineAccounts.get(timestamp);
 
-        if (onlineAccounts != null) {
-            // Remove accounts with matching timestamp, nonce, and public key
-            final Set<OnlineAccountData> finalOnlineAccounts = onlineAccounts;
-            blocksOnlineAccounts.removeIf(a1 -> finalOnlineAccounts.stream()
-                    .anyMatch(a2 -> a2.getTimestamp() == a1.getTimestamp() &&
-                            Objects.equals(a2.getNonce(), a1.getNonce()) &&
-                            Arrays.equals(a2.getPublicKey(), a1.getPublicKey())));
-        }
+        if (onlineAccounts != null)
+            blocksOnlineAccounts.removeAll(onlineAccounts);
     }
 
     /**

src/main/java/org/qortal/controller/PirateChainWalletController.java

@@ -187,7 +187,7 @@ public class PirateChainWalletController extends Thread {
 
             // Check its status
             ArbitraryResourceStatus status = ArbitraryTransactionUtils.getStatus(
-                    t.getService(), t.getName(), t.getIdentifier(), false);
+                    t.getService(), t.getName(), t.getIdentifier(), false, true);
 
             if (status.getStatus() != ArbitraryResourceStatus.Status.READY) {
                 LOGGER.info("Not ready yet: {}", status.getTitle());

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java

@@ -5,13 +5,17 @@ import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
 import org.qortal.arbitrary.exception.MissingDataException;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.repository.DataException;
+import org.qortal.utils.ArbitraryTransactionUtils;
 import org.qortal.utils.NTP;
 
 import java.io.IOException;
 import java.util.Comparator;
 import java.util.Map;
 
+import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status.*;
+
 
 public class ArbitraryDataBuilderThread implements Runnable {
 
@@ -69,6 +73,14 @@ public class ArbitraryDataBuilderThread implements Runnable {
                         continue;
                     }
 
+                    // Get status before build
+                    ArbitraryResourceStatus arbitraryResourceStatus = ArbitraryTransactionUtils.getStatus(queueItem.getService(), queueItem.getResourceId(), queueItem.getIdentifier(), false, true);
+                    if (arbitraryResourceStatus.getStatus() == NOT_PUBLISHED) {
+                        // No point in building a non-existent resource
+                        this.removeFromQueue(queueItem);
+                        continue;
+                    }
+
                     // Set the start timestamp, to prevent other threads from building it at the same time
                     queueItem.prepareForBuild();
                 }

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCacheManager.java

@@ -0,0 +1,236 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.api.resource.TransactionsResource;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.gui.SplashFrame;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.Transaction;
+import org.qortal.utils.Base58;
+
+import java.util.*;
+
+public class ArbitraryDataCacheManager extends Thread {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCacheManager.class);
+
+    private static ArbitraryDataCacheManager instance;
+    private volatile boolean isStopping = false;
+
+    /** Queue of arbitrary transactions that require cache updates */
+    private final List<ArbitraryTransactionData> updateQueue = Collections.synchronizedList(new ArrayList<>());
+
+
+    public static synchronized ArbitraryDataCacheManager getInstance() {
+        if (instance == null) {
+            instance = new ArbitraryDataCacheManager();
+        }
+
+        return instance;
+    }
+
+    @Override
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data Cache Manager");
+
+        try {
+            while (!Controller.isStopping()) {
+                Thread.sleep(500L);
+
+                // Process queue
+                processResourceQueue();
+            }
+        } catch (InterruptedException e) {
+            // Fall through to exit thread
+        }
+
+        // Clear queue before terminating thread
+        processResourceQueue();
+    }
+
+    public void shutdown() {
+        isStopping = true;
+        this.interrupt();
+    }
+
+
+    private void processResourceQueue() {
+        if (this.updateQueue.isEmpty()) {
+            // Nothing to do
+            return;
+        }
+
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            // Take a snapshot of resourceQueue, so we don't need to lock it while processing
+            List<ArbitraryTransactionData> resourceQueueCopy = List.copyOf(this.updateQueue);
+
+            for (ArbitraryTransactionData transactionData : resourceQueueCopy) {
+                // Best not to return when controller is stopping, as ideally we need to finish processing
+
+                LOGGER.debug(() -> String.format("Processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
+
+                // Remove from the queue regardless of outcome
+                this.updateQueue.remove(transactionData);
+
+                // Update arbitrary resource caches
+                try {
+                    ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+                    arbitraryTransaction.updateArbitraryResourceCache(repository);
+                    arbitraryTransaction.updateArbitraryMetadataCache(repository);
+                    repository.saveChanges();
+
+                    // Update status as separate commit, as this is more prone to failure
+                    arbitraryTransaction.updateArbitraryResourceStatus(repository);
+                    repository.saveChanges();
+
+                    LOGGER.debug(() -> String.format("Finished processing transaction %.8s in arbitrary resource queue...", Base58.encode(transactionData.getSignature())));
+
+                } catch (DataException e) {
+                    repository.discardChanges();
+                    LOGGER.error("Repository issue while updating arbitrary resource caches", e);
+                }
+            }
+        } catch (DataException e) {
+            LOGGER.error("Repository issue while processing arbitrary resource cache updates", e);
+        }
+    }
+
+    public void addToUpdateQueue(ArbitraryTransactionData transactionData) {
+        this.updateQueue.add(transactionData);
+        LOGGER.debug(() -> String.format("Transaction %.8s added to queue", Base58.encode(transactionData.getSignature())));
+    }
+
+    public boolean needsArbitraryResourcesCacheRebuild(Repository repository) throws DataException {
+        // Check if we have an entry in the cache for the oldest ARBITRARY transaction with a name
+        List<ArbitraryTransactionData> oldestCacheableTransactions = repository.getArbitraryRepository().getArbitraryTransactions(true, 1, 0, false);
+        if (oldestCacheableTransactions == null || oldestCacheableTransactions.isEmpty()) {
+            // No relevant arbitrary transactions yet on this chain
+            LOGGER.debug("No relevant arbitrary transactions exist to build cache from");
+            return false;
+        }
+        // We have an arbitrary transaction, so check if it's in the cache
+        ArbitraryTransactionData txn = oldestCacheableTransactions.get(0);
+        ArbitraryResourceData cachedResource = repository.getArbitraryRepository().getArbitraryResource(txn.getService(), txn.getName(), txn.getIdentifier());
+        if (cachedResource != null) {
+            // Earliest resource exists in the cache, so assume it has been built.
+            // We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
+            // we shouldn't ever be left in a partially rebuilt state.
+            LOGGER.debug("Arbitrary resources cache already built");
+            return false;
+        }
+
+        return true;
+    }
+
+    public boolean buildArbitraryResourcesCache(Repository repository, boolean forceRebuild) throws DataException {
+        if (Settings.getInstance().isLite()) {
+            // Lite nodes have no blockchain
+            return false;
+        }
+
+        try {
+            // Skip if already built
+            if (!needsArbitraryResourcesCacheRebuild(repository) && !forceRebuild) {
+                LOGGER.debug("Arbitrary resources cache already built");
+                return false;
+            }
+
+            LOGGER.info("Building arbitrary resources cache...");
+            SplashFrame.getInstance().updateStatus("Building QDN cache - please wait...");
+
+            final int batchSize = 100;
+            int offset = 0;
+
+            // Loop through all ARBITRARY transactions, and determine latest state
+            while (!Controller.isStopping()) {
+                LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
+
+                List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
+                if (signatures.isEmpty()) {
+                    // Complete
+                    break;
+                }
+
+                // Expand signatures to transactions
+                for (byte[] signature : signatures) {
+                    ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
+                            .getTransactionRepository().fromSignature(signature);
+
+                    if (transactionData.getService() == null) {
+                        // Unsupported service - ignore this resource
+                        continue;
+                    }
+
+                    // Update arbitrary resource caches
+                    ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+                    arbitraryTransaction.updateArbitraryResourceCache(repository);
+                    arbitraryTransaction.updateArbitraryMetadataCache(repository);
+                    repository.saveChanges();
+                }
+                offset += batchSize;
+            }
+
+            // Now refresh all statuses
+            refreshArbitraryStatuses(repository);
+
+            LOGGER.info("Completed build of arbitrary resources cache.");
+            return true;
+        }
+        catch (DataException e) {
+            LOGGER.info("Unable to build arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage());
+
+            // Throw an exception so that the node startup is halted, allowing for a retry next time.
+            repository.discardChanges();
+            throw new DataException("Build of arbitrary resources cache failed.");
+        }
+    }
+
+    private boolean refreshArbitraryStatuses(Repository repository) throws DataException {
+        try {
+            LOGGER.info("Refreshing arbitrary resource statuses for locally hosted transactions...");
+            SplashFrame.getInstance().updateStatus("Refreshing statuses - please wait...");
+
+            final int batchSize = 100;
+            int offset = 0;
+
+            // Loop through all ARBITRARY transactions, and determine latest state
+            while (!Controller.isStopping()) {
+                LOGGER.info("Fetching hosted transactions {} - {}", offset, offset+batchSize-1);
+
+                List<ArbitraryTransactionData> hostedTransactions = ArbitraryDataStorageManager.getInstance().listAllHostedTransactions(repository, batchSize, offset);
+                if (hostedTransactions.isEmpty()) {
+                    // Complete
+                    break;
+                }
+
+                // Loop through hosted transactions
+                for (ArbitraryTransactionData transactionData : hostedTransactions) {
+
+                    // Determine status and update cache
+                    ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+                    arbitraryTransaction.updateArbitraryResourceStatus(repository);
+                    repository.saveChanges();
+                }
+                offset += batchSize;
+            }
+
+            LOGGER.info("Completed refresh of arbitrary resource statuses.");
+            return true;
+        }
+        catch (DataException e) {
+            LOGGER.info("Unable to refresh arbitrary resource statuses: {}. The database may have been left in an inconsistent state.", e.getMessage());
+
+            // Throw an exception so that the node startup is halted, allowing for a retry next time.
+            repository.discardChanges();
+            throw new DataException("Refresh of arbitrary resource statuses failed.");
+        }
+    }
+
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataFileManager.java

@@ -146,7 +146,7 @@ public class ArbitraryDataFileManager extends Thread {
                 if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
                     LOGGER.debug("Requesting data file {} from peer {}", hash58, peer);
                     Long startTime = NTP.getTime();
-                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, signature, hash, null);
+                    ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, null, arbitraryTransactionData, signature, hash, null);
                     Long endTime = NTP.getTime();
                     if (receivedArbitraryDataFile != null) {
                         LOGGER.debug("Received data file {} from peer {}. Time taken: {} ms", receivedArbitraryDataFile.getHash58(), peer, (endTime-startTime));
@@ -191,7 +191,7 @@ public class ArbitraryDataFileManager extends Thread {
         return receivedAtLeastOneFile;
     }
 
-    private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
+    private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, Peer requestingPeer, ArbitraryTransactionData arbitraryTransactionData, byte[] signature, byte[] hash, Message originalMessage) throws DataException {
         ArbitraryDataFile existingFile = ArbitraryDataFile.fromHash(hash, signature);
         boolean fileAlreadyExists = existingFile.exists();
         String hash58 = Base58.encode(hash);
@@ -250,6 +250,13 @@ public class ArbitraryDataFileManager extends Thread {
             }
         }
 
+        // If this is a metadata file then we need to update the cache
+        if (arbitraryTransactionData != null && arbitraryTransactionData.getMetadataHash() != null) {
+            if (Arrays.equals(arbitraryTransactionData.getMetadataHash(), hash)) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+        }
+
         return arbitraryDataFile;
     }
 
@@ -585,7 +592,9 @@ public class ArbitraryDataFileManager extends Thread {
 
                     // Forward the message to this peer
                     LOGGER.debug("Asking peer {} for hash {}", peerToAsk, hash58);
-                    this.fetchArbitraryDataFile(peerToAsk, peer, signature, hash, message);
+                    // No need to pass arbitraryTransactionData below because this is only used for metadata caching,
+                    // and metadata isn't retained when relaying.
+                    this.fetchArbitraryDataFile(peerToAsk, peer, null, signature, hash, message);
                 }
                 else {
                     LOGGER.debug("Peer {} not found in relay info", peer);

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource;
 import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
@@ -539,6 +540,17 @@ public class ArbitraryDataManager extends Thread {
 		return true;
 	}
 
+	public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) {
+		if (arbitraryTransactionData.getName() == null) {
+			// No name, so we don't care about this transaction
+			return;
+		}
+
+		// Add to queue for update/deletion
+		ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+
+	}
+
 	public int getPowDifficulty() {
 		return this.powDifficulty;
 	}

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -320,41 +320,46 @@ public class ArbitraryMetadataManager {
             return;
         }
 
-        // Update requests map to reflect that we've received all chunks
+        // Update requests map to reflect that we've received this metadata
         Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
         arbitraryMetadataRequests.put(message.getId(), newEntry);
 
-        ArbitraryTransactionData arbitraryTransactionData = null;
-
-        // Forwarding
-        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
-
-            // Get transaction info
-            try (final Repository repository = RepositoryManager.getRepository()) {
-                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-                if (!(transactionData instanceof ArbitraryTransactionData))
-                    return;
-                arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
-            } catch (DataException e) {
-                LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
+        // Get transaction info
+        try (final Repository repository = RepositoryManager.getRepository()) {
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+            if (!(transactionData instanceof ArbitraryTransactionData)) {
+                return;
             }
+            ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-            // Check if the name is blocked
-            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
-            if (!isBlocked) {
-                Peer requestingPeer = request.getB();
-                if (requestingPeer != null) {
+            // Forwarding
+            if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
 
-                    ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
-                    forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
+                // Check if the name is blocked
+                boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+                if (!isBlocked) {
+                    Peer requestingPeer = request.getB();
+                    if (requestingPeer != null) {
 
-                    // Forward to requesting peer
-                    LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
-                    if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
-                        requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
+                        forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
+
+                        // Forward to requesting peer
+                        LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                        if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+                            requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        }
                     }
                 }
             }
+
+            // Add to resource queue to update arbitrary resource caches
+            if (arbitraryTransactionData != null) {
+                ArbitraryDataCacheManager.getInstance().addToUpdateQueue(arbitraryTransactionData);
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while saving arbitrary transaction metadata from peer %s", peer), e);
         }
     }
 

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java

@@ -6,8 +6,10 @@ import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
 import java.util.Objects;
 
+import static org.qortal.data.arbitrary.ArbitraryResourceStatus.Status;
+
 @XmlAccessorType(XmlAccessType.FIELD)
-public class ArbitraryResourceInfo {
+public class ArbitraryResourceData {
 
 	public String name;
 	public Service service;
@@ -15,11 +17,21 @@ public class ArbitraryResourceInfo {
 	public ArbitraryResourceStatus status;
 	public ArbitraryResourceMetadata metadata;
 
-	public Long size;
+	public Integer size;
 	public Long created;
 	public Long updated;
 
-	public ArbitraryResourceInfo() {
+	public ArbitraryResourceData() {
+	}
+
+	public ArbitraryResourceData(Service service, String name, String identifier) {
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		this.service = service;
+		this.name = name;
+		this.identifier = identifier;
 	}
 
 	@Override
@@ -27,15 +39,24 @@ public class ArbitraryResourceInfo {
 		return String.format("%s %s %s", name, service, identifier);
 	}
 
+	public void setStatus(Status status) {
+		if (status == null) {
+			this.status = null;
+		}
+		else {
+			this.status = new ArbitraryResourceStatus(status);
+		}
+	}
+
 	@Override
 	public boolean equals(Object o) {
 		if (o == this)
 			return true;
 
-		if (!(o instanceof ArbitraryResourceInfo))
+		if (!(o instanceof ArbitraryResourceData))
 			return false;
 
-		ArbitraryResourceInfo other = (ArbitraryResourceInfo) o;
+		ArbitraryResourceData other = (ArbitraryResourceData) o;
 
 		return Objects.equals(this.name, other.name) &&
 				Objects.equals(this.service, other.service) &&

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java

@@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata {
     private List<String> files;
     private String mimeType;
 
+    // Only included when updating database
+    private ArbitraryResourceData arbitraryResourceData;
+
     public ArbitraryResourceMetadata() {
     }
 
@@ -60,4 +63,52 @@ public class ArbitraryResourceMetadata {
     public List<String> getFiles() {
         return this.files;
     }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public String getTitle() {
+        return this.title;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setCategory(Category category) {
+        this.category = category;
+
+        // Also set categoryName
+        if (category != null) {
+            this.categoryName = category.getName();
+        }
+    }
+
+    public Category getCategory() {
+        return this.category;
+    }
+
+    public boolean hasMetadata() {
+        return title != null || description != null || tags != null || category != null || files != null || mimeType != null;
+    }
+
+    public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) {
+        this.arbitraryResourceData = arbitraryResourceData;
+    }
+    public ArbitraryResourceData getArbitraryResourceData() {
+        return this.arbitraryResourceData;
+    }
 }

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java

@@ -9,7 +9,7 @@ import java.util.List;
 public class ArbitraryResourceNameInfo {
 
 	public String name;
-	public List<ArbitraryResourceInfo> resources = new ArrayList<>();
+	public List<ArbitraryResourceData> resources = new ArrayList<>();
 
 	public ArbitraryResourceNameInfo() {
 	}

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java

@@ -2,29 +2,46 @@ package org.qortal.data.arbitrary;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
+import java.util.Map;
+
+import static java.util.Arrays.stream;
+import static java.util.stream.Collectors.toMap;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 public class ArbitraryResourceStatus {
 
     public enum Status {
-        PUBLISHED("Published", "Published but not yet downloaded"),
-        NOT_PUBLISHED("Not published", "Resource does not exist"),
-        DOWNLOADING("Downloading", "Locating and downloading files..."),
-        DOWNLOADED("Downloaded", "Files downloaded"),
-        BUILDING("Building", "Building..."),
-        READY("Ready", "Ready"),
-        MISSING_DATA("Missing data", "Unable to locate all files. Please try again later"),
-        BUILD_FAILED("Build failed", "Build failed. Please try again later"),
-        UNSUPPORTED("Unsupported", "Unsupported request"),
-        BLOCKED("Blocked", "Name is blocked so content cannot be served");
+        // Note: integer values must not be updated, as they are stored in the db
+        PUBLISHED(1, "Published", "Published but not yet downloaded"),
+        NOT_PUBLISHED(2, "Not published", "Resource does not exist"),
+        DOWNLOADING(3, "Downloading", "Locating and downloading files..."),
+        DOWNLOADED(4, "Downloaded", "Files downloaded"),
+        BUILDING(5, "Building", "Building..."),
+        READY(6, "Ready", "Ready"),
+        MISSING_DATA(7, "Missing data", "Unable to locate all files. Please try again later"),
+        BUILD_FAILED(8, "Build failed", "Build failed. Please try again later"),
+        UNSUPPORTED(9, "Unsupported", "Unsupported request"),
+        BLOCKED(10, "Blocked", "Name is blocked so content cannot be served");
 
+        public int value;
         private String title;
         private String description;
 
-        Status(String title, String description) {
+        private static final Map<Integer, Status> map = stream(Status.values())
+                .collect(toMap(status -> status.value, status -> status));
+
+        Status(int value, String title, String description) {
+            this.value = value;
             this.title = title;
             this.description = description;
         }
+
+        public static Status valueOf(Integer value) {
+            if (value == null) {
+                return null;
+            }
+            return map.get(value);
+        }
     }
 
     private Status status;

src/main/java/org/qortal/data/network/OnlineAccountData.java

@@ -1,6 +1,7 @@
 package org.qortal.data.network;
 
 import java.util.Arrays;
+import java.util.Objects;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
@@ -34,10 +35,6 @@ public class OnlineAccountData {
 		this.nonce = nonce;
 	}
 
-	public OnlineAccountData(long timestamp, byte[] signature, byte[] publicKey) {
-		this(timestamp, signature, publicKey, null);
-	}
-
 	public long getTimestamp() {
 		return this.timestamp;
 	}
@@ -76,6 +73,10 @@ public class OnlineAccountData {
 		if (otherOnlineAccountData.timestamp != this.timestamp)
 			return false;
 
+		// Almost as quick
+		if (!Objects.equals(otherOnlineAccountData.nonce, this.nonce))
+			return false;
+
 		if (!Arrays.equals(otherOnlineAccountData.publicKey, this.publicKey))
 			return false;
 
@@ -88,9 +89,10 @@ public class OnlineAccountData {
 	public int hashCode() {
 		int h = this.hash;
 		if (h == 0) {
-			this.hash = h = Long.hashCode(this.timestamp)
-					^ Arrays.hashCode(this.publicKey);
+			h = Objects.hash(timestamp, nonce);
+			h = 31 * h + Arrays.hashCode(publicKey);
 			// We don't use signature because newer aggregate signatures use random nonces
+			this.hash = h;
 		}
 		return h;
 	}

src/main/java/org/qortal/network/Network.java

@@ -187,7 +187,7 @@ public class Network {
 
                 this.bindAddress = bindAddress; // Store the selected address, so that it can be used by other parts of the app
                 break; // We don't want to bind to more than one address
-            } catch (UnknownHostException e) {
+            } catch (UnknownHostException | UnsupportedAddressTypeException e) {
                 LOGGER.error("Can't bind listen socket to address {}", Settings.getInstance().getBindAddress());
                 if (i == bindAddresses.size()-1) { // Only throw an exception if all addresses have been tried
                     throw new IOException("Can't bind listen socket to address", e);

src/main/java/org/qortal/network/message/GetOnlineAccountsMessage.java

@@ -1,69 +0,0 @@
-package org.qortal.network.message;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-
-public class GetOnlineAccountsMessage extends Message {
-	private static final int MAX_ACCOUNT_COUNT = 5000;
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public GetOnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.GET_ONLINE_ACCOUNTS);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-
-		try {
-			bytes.write(Ints.toByteArray(onlineAccounts.size()));
-
-			for (OnlineAccountData onlineAccountData : onlineAccounts) {
-				bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
-
-				bytes.write(onlineAccountData.getPublicKey());
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private GetOnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.GET_ONLINE_ACCOUNTS);
-
-		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		final int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
-			long timestamp = bytes.getLong();
-
-			byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-			bytes.get(publicKey);
-
-			onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
-		}
-
-		return new GetOnlineAccountsMessage(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/GetOnlineAccountsV2Message.java

@@ -1,109 +0,0 @@
-package org.qortal.network.message;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * For requesting online accounts info from remote peer, given our list of online accounts.
- *
- * Different format to V1:
- * V1 is: number of entries, then timestamp + pubkey for each entry
- * V2 is: groups of: number of entries, timestamp, then pubkey for each entry
- *
- * Also V2 only builds online accounts message once!
- */
-public class GetOnlineAccountsV2Message extends Message {
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public GetOnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.GET_ONLINE_ACCOUNTS_V2);
-
-		// If we don't have ANY online accounts then it's an easier construction...
-		if (onlineAccounts.isEmpty()) {
-			// Always supply a number of accounts
-			this.dataBytes = Ints.toByteArray(0);
-			this.checksumBytes = Message.generateChecksum(this.dataBytes);
-			return;
-		}
-
-		// How many of each timestamp
-		Map<Long, Integer> countByTimestamp = new HashMap<>();
-
-		for (OnlineAccountData onlineAccountData : onlineAccounts) {
-			Long timestamp = onlineAccountData.getTimestamp();
-			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
-		}
-
-		// We should know exactly how many bytes to allocate now
-		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
-				+ onlineAccounts.size() * Transformer.PUBLIC_KEY_LENGTH;
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
-
-		try {
-			for (long timestamp : countByTimestamp.keySet()) {
-				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
-
-				bytes.write(Longs.toByteArray(timestamp));
-
-				for (OnlineAccountData onlineAccountData : onlineAccounts) {
-					if (onlineAccountData.getTimestamp() == timestamp)
-						bytes.write(onlineAccountData.getPublicKey());
-				}
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private GetOnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.GET_ONLINE_ACCOUNTS_V2);
-
-		this.onlineAccounts = onlineAccounts;
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		while (accountCount > 0) {
-			long timestamp = bytes.getLong();
-
-			for (int i = 0; i < accountCount; ++i) {
-				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-				bytes.get(publicKey);
-
-				onlineAccounts.add(new OnlineAccountData(timestamp, null, publicKey));
-			}
-
-			if (bytes.hasRemaining()) {
-				accountCount = bytes.getInt();
-			} else {
-				// we've finished
-				accountCount = 0;
-			}
-		}
-
-		return new GetOnlineAccountsV2Message(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/MessageType.java

@@ -43,11 +43,7 @@ public enum MessageType {
     BLOCK_SUMMARIES(70, BlockSummariesMessage::fromByteBuffer),
     GET_BLOCK_SUMMARIES(71, GetBlockSummariesMessage::fromByteBuffer),
     BLOCK_SUMMARIES_V2(72, BlockSummariesV2Message::fromByteBuffer),
-
-    ONLINE_ACCOUNTS(80, OnlineAccountsMessage::fromByteBuffer),
-    GET_ONLINE_ACCOUNTS(81, GetOnlineAccountsMessage::fromByteBuffer),
-    ONLINE_ACCOUNTS_V2(82, OnlineAccountsV2Message::fromByteBuffer),
-    GET_ONLINE_ACCOUNTS_V2(83, GetOnlineAccountsV2Message::fromByteBuffer),
+    
     ONLINE_ACCOUNTS_V3(84, OnlineAccountsV3Message::fromByteBuffer),
     GET_ONLINE_ACCOUNTS_V3(85, GetOnlineAccountsV3Message::fromByteBuffer),
 

src/main/java/org/qortal/network/message/OnlineAccountsMessage.java

@@ -1,75 +0,0 @@
-package org.qortal.network.message;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.Collectors;
-
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-
-public class OnlineAccountsMessage extends Message {
-	private static final int MAX_ACCOUNT_COUNT = 5000;
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public OnlineAccountsMessage(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.ONLINE_ACCOUNTS);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream();
-
-		try {
-			bytes.write(Ints.toByteArray(onlineAccounts.size()));
-
-			for (OnlineAccountData onlineAccountData : onlineAccounts) {
-				bytes.write(Longs.toByteArray(onlineAccountData.getTimestamp()));
-
-				bytes.write(onlineAccountData.getSignature());
-
-				bytes.write(onlineAccountData.getPublicKey());
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private OnlineAccountsMessage(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.ONLINE_ACCOUNTS);
-
-		this.onlineAccounts = onlineAccounts.stream().limit(MAX_ACCOUNT_COUNT).collect(Collectors.toList());
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) {
-		final int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		for (int i = 0; i < Math.min(MAX_ACCOUNT_COUNT, accountCount); ++i) {
-			long timestamp = bytes.getLong();
-
-			byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
-			bytes.get(signature);
-
-			byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-			bytes.get(publicKey);
-
-			OnlineAccountData onlineAccountData = new OnlineAccountData(timestamp, signature, publicKey);
-			onlineAccounts.add(onlineAccountData);
-		}
-
-		return new OnlineAccountsMessage(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/OnlineAccountsV2Message.java

@@ -1,113 +0,0 @@
-package org.qortal.network.message;
-
-import com.google.common.primitives.Ints;
-import com.google.common.primitives.Longs;
-import org.qortal.data.network.OnlineAccountData;
-import org.qortal.transform.Transformer;
-
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * For sending online accounts info to remote peer.
- *
- * Different format to V1:
- * V1 is: number of entries, then timestamp + sig + pubkey for each entry
- * V2 is: groups of: number of entries, timestamp, then sig + pubkey for each entry
- *
- * Also V2 only builds online accounts message once!
- */
-public class OnlineAccountsV2Message extends Message {
-
-	private List<OnlineAccountData> onlineAccounts;
-
-	public OnlineAccountsV2Message(List<OnlineAccountData> onlineAccounts) {
-		super(MessageType.ONLINE_ACCOUNTS_V2);
-
-		// Shortcut in case we have no online accounts
-		if (onlineAccounts.isEmpty()) {
-			this.dataBytes = Ints.toByteArray(0);
-			this.checksumBytes = Message.generateChecksum(this.dataBytes);
-			return;
-		}
-
-		// How many of each timestamp
-		Map<Long, Integer> countByTimestamp = new HashMap<>();
-
-		for (OnlineAccountData onlineAccountData : onlineAccounts) {
-			Long timestamp = onlineAccountData.getTimestamp();
-			countByTimestamp.compute(timestamp, (k, v) -> v == null ? 1 : ++v);
-		}
-
-		// We should know exactly how many bytes to allocate now
-		int byteSize = countByTimestamp.size() * (Transformer.INT_LENGTH + Transformer.TIMESTAMP_LENGTH)
-				+ onlineAccounts.size() * (Transformer.SIGNATURE_LENGTH + Transformer.PUBLIC_KEY_LENGTH);
-
-		ByteArrayOutputStream bytes = new ByteArrayOutputStream(byteSize);
-
-		try {
-			for (long timestamp : countByTimestamp.keySet()) {
-				bytes.write(Ints.toByteArray(countByTimestamp.get(timestamp)));
-
-				bytes.write(Longs.toByteArray(timestamp));
-
-				for (OnlineAccountData onlineAccountData : onlineAccounts) {
-					if (onlineAccountData.getTimestamp() == timestamp) {
-						bytes.write(onlineAccountData.getSignature());
-						bytes.write(onlineAccountData.getPublicKey());
-					}
-				}
-			}
-		} catch (IOException e) {
-			throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
-		}
-
-		this.dataBytes = bytes.toByteArray();
-		this.checksumBytes = Message.generateChecksum(this.dataBytes);
-	}
-
-	private OnlineAccountsV2Message(int id, List<OnlineAccountData> onlineAccounts) {
-		super(id, MessageType.ONLINE_ACCOUNTS_V2);
-
-		this.onlineAccounts = onlineAccounts;
-	}
-
-	public List<OnlineAccountData> getOnlineAccounts() {
-		return this.onlineAccounts;
-	}
-
-	public static Message fromByteBuffer(int id, ByteBuffer bytes) throws MessageException {
-		int accountCount = bytes.getInt();
-
-		List<OnlineAccountData> onlineAccounts = new ArrayList<>(accountCount);
-
-		while (accountCount > 0) {
-			long timestamp = bytes.getLong();
-
-			for (int i = 0; i < accountCount; ++i) {
-				byte[] signature = new byte[Transformer.SIGNATURE_LENGTH];
-				bytes.get(signature);
-
-				byte[] publicKey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-				bytes.get(publicKey);
-
-				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey));
-			}
-
-			if (bytes.hasRemaining()) {
-				accountCount = bytes.getInt();
-			} else {
-				// we've finished
-				accountCount = 0;
-			}
-		}
-
-		return new OnlineAccountsV2Message(id, onlineAccounts);
-	}
-
-}

src/main/java/org/qortal/network/message/OnlineAccountsV3Message.java

@@ -99,9 +99,10 @@ public class OnlineAccountsV3Message extends Message {
 				bytes.get(publicKey);
 
 				// Nonce is optional - will be -1 if missing
+				// ... but we should skip/ignore an online account if it has no nonce
 				Integer nonce = bytes.getInt();
 				if (nonce < 0) {
-					nonce = null;
+					continue;
 				}
 
 				onlineAccounts.add(new OnlineAccountData(timestamp, signature, publicKey, nonce));

src/main/java/org/qortal/repository/ArbitraryRepository.java

@@ -1,9 +1,10 @@
 package org.qortal.repository;
 
+import org.qortal.api.SearchMode;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
-import org.qortal.data.network.ArbitraryPeerData;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
 
@@ -11,23 +12,45 @@ import java.util.List;
 
 public interface ArbitraryRepository {
 
+	// Utils
+
 	public boolean isDataLocal(byte[] signature) throws DataException;
 
 	public byte[] fetchData(byte[] signature) throws DataException;
 
+
+	// Transaction related
+
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
 
 	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
 
 	public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
 
+	public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
+
 	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
 
+	public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
-	public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
 
-	public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	// Resource related
 
-	public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException;
 
+	public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+	public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+
+	// Arbitrary resources cache save/load
+
+	public void save(ArbitraryResourceData arbitraryResourceData) throws DataException;
+	public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException;
+	public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException;
+
+	public void save(ArbitraryResourceMetadata metadata) throws DataException;
+	public void delete(ArbitraryResourceMetadata metadata) throws DataException;
 }

src/main/java/org/qortal/repository/TransactionRepository.java

@@ -314,7 +314,7 @@ public interface TransactionRepository {
 	 * @return list of transactions, or empty if none.
 	 * @throws DataException
 	 */
-	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException;
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException;
 
 	/**
 	 * Remove transaction from unconfirmed transactions pile.

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -2,10 +2,13 @@ package org.qortal.repository.hsqldb;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.bouncycastle.util.Longs;
+import org.qortal.api.SearchMode;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
 import org.qortal.data.transaction.BaseTransactionData;
@@ -22,6 +25,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
@@ -41,6 +45,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return (ArbitraryTransactionData) transactionData;
 	}
 
+
+	// Utils
+
 	@Override
 	public boolean isDataLocal(byte[] signature) throws DataException {
 		ArbitraryTransactionData transactionData = getTransactionData(signature);
@@ -113,6 +120,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return null;
 	}
 
+
+	// Transaction related
+
 	@Override
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
 		// Already hashed? Nothing to do
@@ -211,8 +221,12 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		}
 	}
 
-	@Override
-	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
+	private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
+		if (name == null || service == null) {
+			// Required fields
+			return null;
+		}
+
 		StringBuilder sql = new StringBuilder(1024);
 
 		sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@@ -228,7 +242,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			sql.append(method.value);
 		}
 
-		sql.append("ORDER BY created_when DESC LIMIT 1");
+		sql.append(" ORDER BY created_when");
+
+		if (firstNotLast) {
+			sql.append(" ASC");
+		}
+		else {
+			sql.append(" DESC");
+		}
+
+		sql.append(" LIMIT 1");
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
 			if (resultSet == null)
@@ -284,22 +307,286 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 	}
 
 	@Override
-	public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names,
-															 boolean defaultResource,  Boolean followedOnly, Boolean excludeBlocked,
+	public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException {
+		return this.getSingleTransaction(name, service, method, identifier, true);
+	}
+
+	@Override
+	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
+		return this.getSingleTransaction(name, service, method, identifier, false);
+	}
+
+	public List<ArbitraryTransactionData> getArbitraryTransactions(boolean requireName, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
+			"tx_group_id, block_height, approval_status, approval_height, " +
+			"version, nonce, service, size, is_data_raw, data, metadata_hash, " +
+			"name, identifier, update_method, secret, compression FROM ArbitraryTransactions " +
+			"JOIN Transactions USING (signature)");
+
+		if (requireName) {
+			sql.append(" WHERE name IS NOT NULL");
+		}
+
+		sql.append(" ORDER BY created_when");
+
+		if (reverse != null && reverse) {
+			sql.append(" DESC");
+		}
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
+			if (resultSet == null)
+				return null;
+
+			do {
+				//TransactionType type = TransactionType.valueOf(resultSet.getInt(1));
+
+				byte[] reference = resultSet.getBytes(2);
+				byte[] signature = resultSet.getBytes(3);
+				byte[] creatorPublicKey = resultSet.getBytes(4);
+				long timestamp = resultSet.getLong(5);
+
+				Long fee = resultSet.getLong(6);
+				if (fee == 0 && resultSet.wasNull())
+					fee = null;
+
+				int txGroupId = resultSet.getInt(7);
+
+				Integer blockHeight = resultSet.getInt(8);
+				if (blockHeight == 0 && resultSet.wasNull())
+					blockHeight = null;
+
+				ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
+				Integer approvalHeight = resultSet.getInt(10);
+				if (approvalHeight == 0 && resultSet.wasNull())
+					approvalHeight = null;
+
+				BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
+
+				int version = resultSet.getInt(11);
+				int nonce = resultSet.getInt(12);
+				int serviceInt = resultSet.getInt(13);
+				int size = resultSet.getInt(14);
+				boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
+				DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
+				byte[] data = resultSet.getBytes(16);
+				byte[] metadataHash = resultSet.getBytes(17);
+				String nameResult = resultSet.getString(18);
+				String identifierResult = resultSet.getString(19);
+				Method method = Method.valueOf(resultSet.getInt(20));
+				byte[] secret = resultSet.getBytes(21);
+				Compression compression = Compression.valueOf(resultSet.getInt(22));
+				// FUTURE: get payments from signature if needed. Avoiding for now to reduce database calls.
+
+				ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
+						version, serviceInt, nonce, size, nameResult, identifierResult, method, secret,
+						compression, data, dataType, metadataHash, null);
+
+				arbitraryTransactionData.add(transactionData);
+			} while (resultSet.next());
+
+			return arbitraryTransactionData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+		}
+	}
+
+
+	// Resource related
+
+	@Override
+	public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		// Name is required
+		if (name == null) {
+			return null;
+		}
+
+		sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE ArbitraryResourcesCache.service = ? AND ArbitraryResourcesCache.name = ?");
+
+		bindParams.add(service.value);
+		bindParams.add(name);
+
+		if (identifier != null) {
+			sql.append(" AND identifier = ?");
+			bindParams.add(identifier);
+		}
+		else {
+			sql.append(" AND identifier IS NULL");
+		}
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return null;
+
+			String nameResult = resultSet.getString(1);
+			Service serviceResult = Service.valueOf(resultSet.getInt(2));
+			String identifierResult = resultSet.getString(3);
+			Integer sizeResult = resultSet.getInt(4);
+			Integer status = resultSet.getInt(5);
+			Long created = resultSet.getLong(6);
+			Long updated = resultSet.getLong(7);
+
+			// Optional metadata fields
+			String title = resultSet.getString(8);
+			String description = resultSet.getString(9);
+			String category = resultSet.getString(10);
+			String tag1 = resultSet.getString(11);
+			String tag2 = resultSet.getString(12);
+			String tag3 = resultSet.getString(13);
+			String tag4 = resultSet.getString(14);
+			String tag5 = resultSet.getString(15);
+
+			if (Objects.equals(identifierResult, "default")) {
+				// Map "default" back to null. This is optional but probably less confusing than returning "default".
+				identifierResult = null;
+			}
+
+			ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+			arbitraryResourceData.name = nameResult;
+			arbitraryResourceData.service = serviceResult;
+			arbitraryResourceData.identifier = identifierResult;
+			arbitraryResourceData.size = sizeResult;
+			arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
+			arbitraryResourceData.created = created;
+			arbitraryResourceData.updated = (updated == 0) ? null : updated;
+
+			ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+			metadata.setTitle(title);
+			metadata.setDescription(description);
+			metadata.setCategory(Category.uncategorizedValueOf(category));
+
+			List<String> tags = new ArrayList<>();
+			if (tag1 != null) tags.add(tag1);
+			if (tag2 != null) tags.add(tag2);
+			if (tag3 != null) tags.add(tag3);
+			if (tag4 != null) tags.add(tag4);
+			if (tag5 != null) tags.add(tag5);
+			metadata.setTags(!tags.isEmpty() ? tags : null);
+
+			if (metadata.hasMetadata()) {
+				arbitraryResourceData.metadata = metadata;
+			}
+
+			return arbitraryResourceData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary resource from repository", e);
+		}
+	}
+	@Override
+	public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE name IS NOT NULL ORDER BY created_when");
+
+		if (reverse != null && reverse) {
+			sql.append(" DESC");
+		}
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return arbitraryResources;
+
+			do {
+				String nameResult = resultSet.getString(1);
+				Service serviceResult = Service.valueOf(resultSet.getInt(2));
+				String identifierResult = resultSet.getString(3);
+				Integer sizeResult = resultSet.getInt(4);
+				Integer status = resultSet.getInt(5);
+				Long created = resultSet.getLong(6);
+				Long updated = resultSet.getLong(7);
+
+				// Optional metadata fields
+				String title = resultSet.getString(8);
+				String description = resultSet.getString(9);
+				String category = resultSet.getString(10);
+				String tag1 = resultSet.getString(11);
+				String tag2 = resultSet.getString(12);
+				String tag3 = resultSet.getString(13);
+				String tag4 = resultSet.getString(14);
+				String tag5 = resultSet.getString(15);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
+				}
+
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+				arbitraryResourceData.name = nameResult;
+				arbitraryResourceData.service = serviceResult;
+				arbitraryResourceData.identifier = identifierResult;
+				arbitraryResourceData.size = sizeResult;
+				arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
+				arbitraryResourceData.created = created;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
+
+				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+				metadata.setTitle(title);
+				metadata.setDescription(description);
+				metadata.setCategory(Category.uncategorizedValueOf(category));
+
+				List<String> tags = new ArrayList<>();
+				if (tag1 != null) tags.add(tag1);
+				if (tag2 != null) tags.add(tag2);
+				if (tag3 != null) tags.add(tag3);
+				if (tag4 != null) tags.add(tag4);
+				if (tag5 != null) tags.add(tag5);
+				metadata.setTags(!tags.isEmpty() ? tags : null);
+
+				if (metadata.hasMetadata()) {
+					arbitraryResourceData.metadata = metadata;
+				}
+
+				arbitraryResources.add(arbitraryResourceData);
+			} while (resultSet.next());
+
+			return arbitraryResources;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
+		}
+	}
+
+	@Override
+	public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names,
+															 boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
+															 Boolean includeMetadata, Boolean includeStatus,
 															 Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(512);
 		List<Object> bindParams = new ArrayList<>();
 
-		sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1");
+		sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE name IS NOT NULL");
 
 		if (service != null) {
-			sql.append(" AND service = ");
-			sql.append(service.value);
+			sql.append(" AND service = ?");
+			bindParams.add(service.value);
 		}
 
 		if (defaultResource) {
 			// Default resource requested - use NULL identifier
-			sql.append(" AND identifier IS NULL");
+			sql.append(" AND identifier='default'");
 		}
 		else {
 			// Non-default resource requested
@@ -351,7 +638,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			}
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
+		sql.append(" ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -359,53 +646,122 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
 
-		List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
-				return null;
+				return arbitraryResources;
 
 			do {
 				String nameResult = resultSet.getString(1);
 				Service serviceResult = Service.valueOf(resultSet.getInt(2));
 				String identifierResult = resultSet.getString(3);
 				Integer sizeResult = resultSet.getInt(4);
+				Integer status = resultSet.getInt(5);
+				Long created = resultSet.getLong(6);
+				Long updated = resultSet.getLong(7);
 
-				// We should filter out resources without names
-				if (nameResult == null) {
-					continue;
+				// Optional metadata fields
+				String title = resultSet.getString(8);
+				String description = resultSet.getString(9);
+				String category = resultSet.getString(10);
+				String tag1 = resultSet.getString(11);
+				String tag2 = resultSet.getString(12);
+				String tag3 = resultSet.getString(13);
+				String tag4 = resultSet.getString(14);
+				String tag5 = resultSet.getString(15);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
 				}
 
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
-				arbitraryResourceInfo.name = nameResult;
-				arbitraryResourceInfo.service = serviceResult;
-				arbitraryResourceInfo.identifier = identifierResult;
-				arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+				arbitraryResourceData.name = nameResult;
+				arbitraryResourceData.service = serviceResult;
+				arbitraryResourceData.identifier = identifierResult;
+				arbitraryResourceData.size = sizeResult;
+				arbitraryResourceData.created = created;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
 
-				arbitraryResources.add(arbitraryResourceInfo);
+				if (includeStatus != null && includeStatus) {
+					arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
+				}
+
+				if (includeMetadata != null && includeMetadata) {
+					// TODO: we could avoid the join altogether
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setTitle(title);
+					metadata.setDescription(description);
+					metadata.setCategory(Category.uncategorizedValueOf(category));
+
+					List<String> tags = new ArrayList<>();
+					if (tag1 != null) tags.add(tag1);
+					if (tag2 != null) tags.add(tag2);
+					if (tag3 != null) tags.add(tag3);
+					if (tag4 != null) tags.add(tag4);
+					if (tag5 != null) tags.add(tag5);
+					metadata.setTags(!tags.isEmpty() ? tags : null);
+
+					if (metadata.hasMetadata()) {
+						arbitraryResourceData.metadata = metadata;
+					}
+				}
+
+				arbitraryResources.add(arbitraryResourceData);
 			} while (resultSet.next());
 
 			return arbitraryResources;
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
 		}
 	}
 
 	@Override
-	public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
-																List<String> exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
-																Integer limit, Integer offset, Boolean reverse) throws DataException {
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, String title, String description, boolean prefixOnly,
+																List<String> exactMatchNames, boolean defaultResource, SearchMode mode, Integer minLevel, Boolean followedOnly, Boolean excludeBlocked,
+																Boolean includeMetadata, Boolean includeStatus, Long before, Long after, Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(512);
 		List<Object> bindParams = new ArrayList<>();
 
-		sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " +
-				"FROM ArbitraryTransactions " +
-				"JOIN Transactions USING (signature) " +
-				"WHERE 1=1");
+		sql.append("SELECT name, service, identifier, size, status, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache");
+
+		// Default to "latest" mode
+		if (mode == null) {
+			mode = SearchMode.LATEST;
+		}
+
+		switch (mode) {
+			case LATEST:
+				// Include latest item only for a name/service combination
+				sql.append(" JOIN (SELECT name, service, MAX(created_when) AS latest " +
+						"FROM ArbitraryResourcesCache GROUP BY name, service) LatestResources " +
+						"ON name=LatestResources.name AND service=LatestResources.service " +
+						"AND created_when=LatestResources.latest");
+				break;
+
+			case ALL:
+				break;
+		}
+
+		if (minLevel != null) {
+			// Join tables necessary for level filter
+			sql.append(" JOIN Names USING (name) JOIN Accounts ON Accounts.account=Names.owner");
+		}
+
+		sql.append(" LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) WHERE name IS NOT NULL");
+
+		if (minLevel != null) {
+			// Add level filter
+			sql.append(" AND Accounts.level >= ?");
+			bindParams.add(minLevel);
+		}
 
 		if (service != null) {
-			sql.append(" AND service = ");
-			sql.append(service.value);
+			sql.append(" AND service = ?");
+			bindParams.add(service.value);
 		}
 
 		// Handle general query matches
@@ -417,14 +773,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 			if (defaultResource) {
 				// Default resource requested - use NULL identifier and search name only
-				sql.append(" AND LCASE(name) LIKE ? AND identifier IS NULL");
+				sql.append(" AND LCASE(name) LIKE ? AND identifier='default'");
 				bindParams.add(queryWildcard);
 			} else {
 				// Non-default resource requested
 				// In this case we search the identifier as well as the name
-				sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ?)");
-				bindParams.add(queryWildcard);
-				bindParams.add(queryWildcard);
+				sql.append(" AND (LCASE(name) LIKE ? OR LCASE(identifier) LIKE ? OR LCASE(title) LIKE ? OR LCASE(description) LIKE ?)");
+				bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard); bindParams.add(queryWildcard);
 			}
 		}
 
@@ -436,6 +791,22 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			bindParams.add(queryWildcard);
 		}
 
+		// Handle title metadata matches
+		if (title != null) {
+			// Search anywhere in the title, unless "prefixOnly" has been requested
+			String queryWildcard = prefixOnly ? String.format("%s%%", title.toLowerCase()) : String.format("%%%s%%", title.toLowerCase());
+			sql.append(" AND LCASE(title) LIKE ?");
+			bindParams.add(queryWildcard);
+		}
+
+		// Handle description metadata matches
+		if (description != null) {
+			// Search anywhere in the description, unless "prefixOnly" has been requested
+			String queryWildcard = prefixOnly ? String.format("%s%%", description.toLowerCase()) : String.format("%%%s%%", description.toLowerCase());
+			sql.append(" AND LCASE(description) LIKE ?");
+			bindParams.add(queryWildcard);
+		}
+
 		// Handle name searches
 		if (names != null && !names.isEmpty()) {
 			sql.append(" AND (");
@@ -462,6 +833,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			sql.append(")");
 		}
 
+		// Timestamp range
+		if (before != null) {
+			sql.append(" AND created_when < ?");
+			bindParams.add(before);
+		}
+		if (after != null) {
+			sql.append(" AND created_when > ?");
+			bindParams.add(after);
+		}
+
 		// Handle "followed only"
 		if (followedOnly != null && followedOnly) {
 			List<String> followedNames = ListUtils.followedNames();
@@ -492,7 +873,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			}
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY date_created");
+		sql.append(" ORDER BY created_when");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -500,98 +881,182 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
 
-		List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
-				return null;
+				return arbitraryResources;
 
 			do {
 				String nameResult = resultSet.getString(1);
 				Service serviceResult = Service.valueOf(resultSet.getInt(2));
 				String identifierResult = resultSet.getString(3);
 				Integer sizeResult = resultSet.getInt(4);
-				long dateCreated = resultSet.getLong(5);
-				long dateUpdated = resultSet.getLong(6);
+				Integer status = resultSet.getInt(5);
+				Long created = resultSet.getLong(6);
+				Long updated = resultSet.getLong(7);
 
-				// We should filter out resources without names
-				if (nameResult == null) {
-					continue;
+				// Optional metadata fields
+				String titleResult = resultSet.getString(8);
+				String descriptionResult = resultSet.getString(9);
+				String category = resultSet.getString(10);
+				String tag1 = resultSet.getString(11);
+				String tag2 = resultSet.getString(12);
+				String tag3 = resultSet.getString(13);
+				String tag4 = resultSet.getString(14);
+				String tag5 = resultSet.getString(15);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
 				}
 
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
-				arbitraryResourceInfo.name = nameResult;
-				arbitraryResourceInfo.service = serviceResult;
-				arbitraryResourceInfo.identifier = identifierResult;
-				arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
-				arbitraryResourceInfo.created = dateCreated;
-				arbitraryResourceInfo.updated = dateUpdated;
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+				arbitraryResourceData.name = nameResult;
+				arbitraryResourceData.service = serviceResult;
+				arbitraryResourceData.identifier = identifierResult;
+				arbitraryResourceData.size = sizeResult;
+				arbitraryResourceData.created = created;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
 
-				arbitraryResources.add(arbitraryResourceInfo);
+				if (includeStatus != null && includeStatus) {
+					arbitraryResourceData.setStatus(ArbitraryResourceStatus.Status.valueOf(status));
+				}
+
+				if (includeMetadata != null && includeMetadata) {
+					// TODO: we could avoid the join altogether
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setTitle(titleResult);
+					metadata.setDescription(descriptionResult);
+					metadata.setCategory(Category.uncategorizedValueOf(category));
+
+					List<String> tags = new ArrayList<>();
+					if (tag1 != null) tags.add(tag1);
+					if (tag2 != null) tags.add(tag2);
+					if (tag3 != null) tags.add(tag3);
+					if (tag4 != null) tags.add(tag4);
+					if (tag5 != null) tags.add(tag5);
+					metadata.setTags(!tags.isEmpty() ? tags : null);
+
+					if (metadata.hasMetadata()) {
+						arbitraryResourceData.metadata = metadata;
+					}
+				}
+
+				arbitraryResources.add(arbitraryResourceData);
 			} while (resultSet.next());
 
 			return arbitraryResources;
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
+		}
+	}
+
+
+	// Arbitrary resources cache save/load
+
+	@Override
+	public void save(ArbitraryResourceData arbitraryResourceData) throws DataException {
+		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache");
+
+		// "status" isn't saved here as we update this field separately
+		saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
+				.bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size)
+				.bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated);
+
+		try {
+			saveHelper.execute(this.repository);
+		} catch (SQLException e) {
+			throw new DataException("Unable to save arbitrary resource info into repository", e);
 		}
 	}
 
 	@Override
-	public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier,
-																			boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException {
-		StringBuilder sql = new StringBuilder(512);
-
-		sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1");
-
-		if (service != null) {
-			sql.append(" AND service = ");
-			sql.append(service.value);
+	public void setStatus(ArbitraryResourceData arbitraryResourceData, ArbitraryResourceStatus.Status status) throws DataException {
+		if (status == null) {
+			return;
 		}
+		String updateSql = "UPDATE ArbitraryResourcesCache SET status = ? WHERE service = ? AND LCASE(name) = ? AND LCASE(identifier) = ?";
 
-		if (defaultResource) {
-			// Default resource requested - use NULL identifier
-			// The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute()
-			identifier = null;
-			sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)");
-		}
-		else {
-			// Non-default resource requested
-			// Use an exact match identifier, or list all if supplied identifier is null
-			sql.append(" AND (identifier = ? OR (? IS NULL))");
-		}
-
-		sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
-
-		if (reverse != null && reverse) {
-			sql.append(" DESC");
-		}
-
-		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
-
-		List<ArbitraryResourceNameInfo> arbitraryResources = new ArrayList<>();
-
-		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) {
-			if (resultSet == null)
-				return null;
-
-			do {
-				String name = resultSet.getString(1);
-
-				// We should filter out resources without names
-				if (name == null) {
-					continue;
-				}
-
-				ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo();
-				arbitraryResourceNameInfo.name = name;
-
-				arbitraryResources.add(arbitraryResourceNameInfo);
-			} while (resultSet.next());
-
-			return arbitraryResources;
+		try {
+			this.repository.executeCheckedUpdate(updateSql, status.value, arbitraryResourceData.service.value, arbitraryResourceData.name.toLowerCase(), arbitraryResourceData.identifier.toLowerCase());
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to set status for arbitrary resource", e);
 		}
 	}
 
+	@Override
+	public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException {
+		// NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE"
+		// in ArbitraryMetadataCache' FOREIGN KEY definition.
+		try {
+			this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?",
+					arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
+
+		} catch (SQLException e) {
+			throw new DataException("Unable to delete account from repository", e);
+		}
+	}
+
+
+	/* Arbitrary metadata cache */
+
+	@Override
+	public void save(ArbitraryResourceMetadata metadata) throws DataException {
+		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache");
+
+		ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
+		if (arbitraryResourceData == null) {
+			throw new DataException("Can't save metadata without a referenced resource");
+		}
+
+		// Trim metadata values if they are too long to fit in the db
+		String title = ArbitraryDataTransactionMetadata.limitTitle(metadata.getTitle());
+		String description = ArbitraryDataTransactionMetadata.limitDescription(metadata.getDescription());
+		List<String> tags = ArbitraryDataTransactionMetadata.limitTags(metadata.getTags());
+
+		String tag1 = null;
+		String tag2 = null;
+		String tag3 = null;
+		String tag4 = null;
+		String tag5 = null;
+
+		if (tags != null) {
+			if (tags.size() > 0) tag1 = tags.get(0);
+			if (tags.size() > 1) tag2 = tags.get(1);
+			if (tags.size() > 2) tag3 = tags.get(2);
+			if (tags.size() > 3) tag4 = tags.get(3);
+			if (tags.size() > 4) tag5 = tags.get(4);
+		}
+
+		String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null;
+
+		saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
+				.bind("identifier", arbitraryResourceData.identifier).bind("title", title)
+				.bind("description", description).bind("category", category)
+				.bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4)
+				.bind("tag5", tag5);
+
+		try {
+			saveHelper.execute(this.repository);
+		} catch (SQLException e) {
+			throw new DataException("Unable to save arbitrary metadata into repository", e);
+		}
+	}
+
+	@Override
+	public void delete(ArbitraryResourceMetadata metadata) throws DataException {
+		ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
+		if (arbitraryResourceData == null) {
+			throw new DataException("Can't delete metadata without a referenced resource");
+		}
+
+		try {
+			this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?",
+					arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
+
+		} catch (SQLException e) {
+			throw new DataException("Unable to delete account from repository", e);
+		}
+	}
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates {
 				case 37:
 					// ARBITRARY transaction updates for off-chain data storage
 
-					// We may want to use a nonce rather than a transaction fee on the data chain
+					// We may want to use a nonce rather than a transaction fee for ARBITRARY transactions
 					stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0");
 					// We need to know the total size of the data file(s) associated with each transaction
 					stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0");
@@ -1004,6 +1004,49 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("CREATE INDEX TransactionHeightSequenceIndex on Transactions (block_height, block_sequence)");
 					break;
 
+				case 48:
+					// We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching.
+					// IMPORTANT: this is a cache of the last known state of a resource (both confirmed
+					// and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a
+					// given block height, and therefore must NOT be used for any consensus/validation code. It is
+					// simply a cache, to avoid having to query the raw transactions and the metadata in flat files
+					// when serving API requests.
+					// ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update
+					// names or groups for instance, as there is no distinction between creations and updates, and metadata
+					// is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very
+					// easy to keep a cache of the latest transaction's data, but anything more than that would need
+					// considerable thought (and most likely a rewrite).
+
+					stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, "
+							+ "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, "
+							+ "status INTEGER, created_when EpochMillis NOT NULL, updated_when EpochMillis, "
+							+ "PRIMARY KEY (service, name, identifier))");
+					// For finding resources by service.
+					stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)");
+					// For finding resources by name.
+					stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)");
+					// For finding resources by identifier.
+					stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)");
+					// For finding resources by creation date (the default column when ordering).
+					stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE");
+
+					stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, "
+							+ "name RegisteredName NOT NULL, identifier VARCHAR(64), "
+							+ "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), "
+							+ "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), "
+							+ "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) "
+							+ "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)");
+					// For finding metadata by title.
+					stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)");
+
+					// For finding arbitrary transactions by service
+					stmt.execute("CREATE INDEX ArbitraryServiceIndex ON ArbitraryTransactions (service)");
+					// For finding arbitrary transactions by identifier
+					stmt.execute("CREATE INDEX ArbitraryIdentifierIndex ON ArbitraryTransactions (identifier)");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBTransactionRepository.java

@@ -1429,8 +1429,10 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 	}
 
 	@Override
-	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes) throws DataException {
+	public List<TransactionData> getUnconfirmedTransactions(EnumSet<TransactionType> excludedTxTypes, Integer limit) throws DataException {
 		StringBuilder sql = new StringBuilder(1024);
+		List<Object> bindParams = new ArrayList<>();
+
 		sql.append("SELECT signature FROM UnconfirmedTransactions ");
 		sql.append("JOIN Transactions USING (signature) ");
 		sql.append("WHERE type NOT IN (");
@@ -1446,12 +1448,17 @@ public class HSQLDBTransactionRepository implements TransactionRepository {
 		}
 
 		sql.append(")");
-		sql.append("ORDER BY created_when, signature");
+		sql.append("ORDER BY created_when, signature ");
+
+		if (limit != null) {
+			sql.append("LIMIT ?");
+			bindParams.add(limit);
+		}
 
 		List<TransactionData> transactions = new ArrayList<>();
 
 		// Find transactions with no corresponding row in BlockTransactions
-		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString())) {
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
 				return transactions;
 

src/main/java/org/qortal/settings/Settings.java

@@ -146,6 +146,9 @@ public class Settings {
 	/* How many blocks to cache locally. Defaulted to 10, which covers a typical Synchronizer request + a few spare */
 	private int blockCacheSize = 10;
 
+	/** Maximum number of transactions for the block minter to include in a block */
+	private int maxTransactionsPerBlock = 25;
+
 	/** How long to keep old, full, AT state data (ms). */
 	private long atStatesMaxLifetime = 5 * 24 * 60 * 60 * 1000L; // milliseconds
 	/** How often to attempt AT state trimming (ms). */
@@ -693,6 +696,10 @@ public class Settings {
 		return this.blockCacheSize;
 	}
 
+	public int getMaxTransactionsPerBlock() {
+		return this.maxTransactionsPerBlock;
+	}
+
 	public boolean isTestNet() {
 		return this.isTestNet;
 	}

src/main/java/org/qortal/transaction/ArbitraryTransaction.java

@@ -1,18 +1,26 @@
 package org.qortal.transaction;
 
-import java.util.Arrays;
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.qortal.account.Account;
+import org.qortal.arbitrary.ArbitraryDataResource;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Service;
 import org.qortal.block.BlockChain;
+import org.qortal.controller.arbitrary.ArbitraryDataCacheManager;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
 import org.qortal.crypto.Crypto;
 import org.qortal.crypto.MemoryPoW;
 import org.qortal.data.PaymentData;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
+import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -20,6 +28,7 @@ import org.qortal.payment.Payment;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.arbitrary.ArbitraryDataFile;
+import org.qortal.repository.RepositoryManager;
 import org.qortal.transform.TransformationException;
 import org.qortal.transform.Transformer;
 import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
@@ -29,6 +38,8 @@ import org.qortal.utils.NTP;
 
 public class ArbitraryTransaction extends Transaction {
 
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryTransaction.class);
+
 	// Properties
 	private ArbitraryTransactionData arbitraryTransactionData;
 
@@ -240,14 +251,8 @@ public class ArbitraryTransaction extends Transaction {
 		// We may need to move files from the misc_ folder
 		ArbitraryTransactionUtils.checkAndRelocateMiscFiles(arbitraryTransactionData);
 
-		// If the data is local, we need to perform a few actions
-		if (isDataLocal()) {
-
-			// We have the data for this transaction, so invalidate the cache
-			if (arbitraryTransactionData.getName() != null) {
-				ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
-			}
-		}
+		// Update caches
+		updateCaches();
 	}
 
 	@Override
@@ -267,6 +272,35 @@ public class ArbitraryTransaction extends Transaction {
 	public void process() throws DataException {
 		// Wrap and delegate payment processing to Payment class.
 		new Payment(this.repository).process(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments());
+
+		// Update caches
+		this.updateCaches();
+	}
+
+	private void updateCaches() {
+		// Very important to use a separate repository instance from the one being used for validation/processing
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			// If the data is local, we need to perform a few actions
+			if (isDataLocal()) {
+
+				// We have the data for this transaction, so invalidate the file cache
+				if (arbitraryTransactionData.getName() != null) {
+					ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
+				}
+			}
+
+			// Add/update arbitrary resource caches, but don't update the status as this involves time-consuming
+			// disk reads, and is more prone to failure. The status will be updated on metadata retrieval, or when
+			// accessing the resource.
+			this.updateArbitraryResourceCache(repository);
+			this.updateArbitraryMetadataCache(repository);
+
+			repository.saveChanges();
+
+		} catch (Exception e) {
+			// Log and ignore all exceptions. The cache is updated from other places too, and can be rebuilt if needed.
+			LOGGER.info("Unable to update arbitrary caches", e);
+		}
 	}
 
 	@Override
@@ -304,4 +338,166 @@ public class ArbitraryTransaction extends Transaction {
 		return null;
 	}
 
+	/**
+	 * Update the arbitrary resources cache.
+	 * This finds the latest transaction and replaces the
+	 * majority of the data in the cache. The current
+	 * transaction is used for the created time,
+	 * if it has a lower timestamp than the existing value.
+	 * It's also used to identify the correct
+	 * service/name/identifier combination.
+	 *
+	 * @throws DataException
+	 */
+	public void updateArbitraryResourceCache(Repository repository) throws DataException {
+		// Don't cache resources without a name (such as auto updates)
+		if (arbitraryTransactionData.getName() == null) {
+			return;
+		}
+
+		Service service = arbitraryTransactionData.getService();
+		String name = arbitraryTransactionData.getName();
+		String identifier = arbitraryTransactionData.getIdentifier();
+
+		if (service == null) {
+			// Unsupported service - ignore this resource
+			return;
+		}
+
+		// In the cache we store null identifiers as "default", as it is part of the primary key
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		// Get the latest transaction
+		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
+		if (latestTransactionData == null) {
+			// We don't have a latest transaction, so delete from cache
+			repository.getArbitraryRepository().delete(arbitraryResourceData);
+			return;
+		}
+
+		// Get existing cached entry if it exists
+		ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
+				.getArbitraryResource(service, name, identifier);
+
+		// Check for existing cached data
+		if (existingArbitraryResourceData == null) {
+			// Nothing exists yet, so set creation date from the current transaction (it will be reduced later if needed)
+			arbitraryResourceData.created = arbitraryTransactionData.getTimestamp();
+			arbitraryResourceData.updated = null;
+		}
+		else {
+			// An entry already exists - update created time from current transaction if this is older
+			arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
+
+			// Set updated time to the latest transaction's timestamp, unless it matches the creation time
+			if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
+				// Latest transaction matches created time, so it hasn't been updated
+				arbitraryResourceData.updated = null;
+			}
+			else {
+				arbitraryResourceData.updated = latestTransactionData.getTimestamp();
+			}
+		}
+
+		arbitraryResourceData.size = latestTransactionData.getSize();
+
+		// Save
+		repository.getArbitraryRepository().save(arbitraryResourceData);
+	}
+
+	public void updateArbitraryResourceStatus(Repository repository) throws DataException {
+		// Don't cache resources without a name (such as auto updates)
+		if (arbitraryTransactionData.getName() == null) {
+			return;
+		}
+
+		Service service = arbitraryTransactionData.getService();
+		String name = arbitraryTransactionData.getName();
+		String identifier = arbitraryTransactionData.getIdentifier();
+
+		if (service == null) {
+			// Unsupported service - ignore this resource
+			return;
+		}
+
+		// In the cache we store null identifiers as "default", as it is part of the primary key
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		// Update status
+		ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
+		ArbitraryResourceStatus arbitraryResourceStatus = resource.getStatus(repository);
+		ArbitraryResourceStatus.Status status = arbitraryResourceStatus != null ? arbitraryResourceStatus.getStatus() : null;
+		repository.getArbitraryRepository().setStatus(arbitraryResourceData, status);
+	}
+
+	public void updateArbitraryMetadataCache(Repository repository) throws DataException {
+		// Get the latest transaction
+		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
+		if (latestTransactionData == null) {
+			// We don't have a latest transaction, so give up
+			return;
+		}
+
+		Service service = latestTransactionData.getService();
+		String name = latestTransactionData.getName();
+		String identifier = latestTransactionData.getIdentifier();
+
+		if (service == null) {
+			// Unsupported service - ignore this resource
+			return;
+		}
+
+		// In the cache we store null identifiers as "default", as it is part of the primary key
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		// Update metadata for latest transaction if it is local
+		if (latestTransactionData.getMetadataHash() != null) {
+			ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
+			if (metadataFile.exists()) {
+				ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+				try {
+					transactionMetadata.read();
+
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setArbitraryResourceData(arbitraryResourceData);
+					metadata.setTitle(transactionMetadata.getTitle());
+					metadata.setDescription(transactionMetadata.getDescription());
+					metadata.setCategory(transactionMetadata.getCategory());
+					metadata.setTags(transactionMetadata.getTags());
+					repository.getArbitraryRepository().save(metadata);
+
+				} catch (IOException e) {
+					// Ignore, as we can add it again later
+				}
+			} else {
+				// We don't have a local copy of this metadata file, so delete it from the cache
+				// It will be re-added if the file later arrives via the network
+				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+				metadata.setArbitraryResourceData(arbitraryResourceData);
+				repository.getArbitraryRepository().delete(metadata);
+			}
+		}
+	}
+
 }

src/main/java/org/qortal/transaction/Transaction.java

@@ -641,7 +641,7 @@ public abstract class Transaction {
 		BlockData latestBlockData = repository.getBlockRepository().getLastBlock();
 
 		EnumSet<TransactionType> excludedTxTypes = EnumSet.of(TransactionType.CHAT, TransactionType.PRESENCE);
-		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes);
+		List<TransactionData> unconfirmedTransactions = repository.getTransactionRepository().getUnconfirmedTransactions(excludedTxTypes, null);
 
 		unconfirmedTransactions.sort(getDataComparator());
 

src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java

@@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.*;
-import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
-import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils {
                                 "chunks if needed", Base58.encode(completeHash));
 
                         ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter);
-                    }
-                    else {
+                    } else {
                         // File might be in use. It's best to leave it and it it will be cleaned up later.
                     }
                 }
@@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils {
      * When first uploaded, files go into a _misc folder as they are not yet associated with a
      * transaction signature. Once the transaction is broadcast, they need to be moved to the
      * correct location, keyed by the transaction signature.
+     *
      * @param arbitraryTransactionData
      * @return
      * @throws DataException
@@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils {
                     file.createNewFile();
                 }
             }
-        }
-        catch (DataException | IOException e) {
+        } catch (DataException | IOException e) {
             LOGGER.info("Unable to check and relocate all files for signature {}: {}",
                     Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
         }
@@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils {
     }
 
     public static List<ArbitraryTransactionData> limitOffsetTransactions(List<ArbitraryTransactionData> transactions,
-                                                                              Integer limit, Integer offset) {
+                                                                         Integer limit, Integer offset) {
         if (limit != null && limit == 0) {
             limit = null;
         }
@@ -389,18 +386,19 @@ public class ArbitraryTransactionUtils {
 
     /**
      * Lookup status of resource
+     *
      * @param service
      * @param name
      * @param identifier
      * @param build
      * @return
      */
-    public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build) {
+    public static ArbitraryResourceStatus getStatus(Service service, String name, String identifier, Boolean build, boolean updateCache) {
 
         // If "build" has been specified, build the resource before returning its status
         if (build != null && build == true) {
-            ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
             try {
+                ArbitraryDataReader reader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
                 if (!reader.isBuilding()) {
                     reader.loadSynchronously(false);
                 }
@@ -410,44 +408,6 @@ public class ArbitraryTransactionUtils {
         }
 
         ArbitraryDataResource resource = new ArbitraryDataResource(name, ArbitraryDataFile.ResourceIdType.NAME, service, identifier);
-        return resource.getStatus(false);
+        return resource.getStatusAndUpdateCache(updateCache);
     }
-
-    public static List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
-        // Determine and add the status of each resource
-        List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
-        for (ArbitraryResourceInfo resourceInfo : resources) {
-            try {
-                ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
-                        resourceInfo.service, resourceInfo.identifier);
-                ArbitraryResourceStatus status = resource.getStatus(true);
-                if (status != null) {
-                    resourceInfo.status = status;
-                }
-                updatedResources.add(resourceInfo);
-
-            } catch (Exception e) {
-                // Catch and log all exceptions, since some systems are experiencing 500 errors when including statuses
-                LOGGER.info("Caught exception when adding status to resource {}: {}", resourceInfo, e.toString());
-            }
-        }
-        return updatedResources;
-    }
-
-    public static List<ArbitraryResourceInfo> addMetadataToResources(List<ArbitraryResourceInfo> resources) {
-        // Add metadata fields to each resource if they exist
-        List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
-        for (ArbitraryResourceInfo resourceInfo : resources) {
-            ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
-                    resourceInfo.service, resourceInfo.identifier);
-            ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata();
-            ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false);
-            if (resourceMetadata != null) {
-                resourceInfo.metadata = resourceMetadata;
-            }
-            updatedResources.add(resourceInfo);
-        }
-        return updatedResources;
-    }
-
 }

src/main/java/org/qortal/utils/StringUtils.java

@@ -0,0 +1,13 @@
+package org.qortal.utils;
+
+public class StringUtils {
+
+    public static String sanitizeString(String input) {
+        String sanitized = input
+                .replaceAll("[<>:\"/\\\\|?*]", "") // Remove invalid characters
+                .replaceAll("^\\s+|\\s+$", "") // Trim leading and trailing whitespace
+                .replaceAll("\\s+", "_"); // Replace consecutive whitespace with underscores
+
+        return sanitized;
+    }
+}

src/main/resources/q-apps/q-apps.js

@@ -227,14 +227,20 @@ window.addEventListener("message", (event) => {
             if (data.identifier != null) url = url.concat("&identifier=" + data.identifier);
             if (data.name != null) url = url.concat("&name=" + data.name);
             if (data.names != null) data.names.forEach((x, i) => url = url.concat("&name=" + x));
+            if (data.title != null) url = url.concat("&title=" + data.title);
+            if (data.description != null) url = url.concat("&description=" + data.description);
             if (data.prefix != null) url = url.concat("&prefix=" + new Boolean(data.prefix).toString());
             if (data.exactMatchNames != null) url = url.concat("&exactmatchnames=" + new Boolean(data.exactMatchNames).toString());
             if (data.default != null) url = url.concat("&default=" + new Boolean(data.default).toString());
+            if (data.mode != null) url = url.concat("&mode=" + data.mode);
+            if (data.minLevel != null) url = url.concat("&minlevel=" + data.minLevel);
             if (data.includeStatus != null) url = url.concat("&includestatus=" + new Boolean(data.includeStatus).toString());
             if (data.includeMetadata != null) url = url.concat("&includemetadata=" + new Boolean(data.includeMetadata).toString());
             if (data.nameListFilter != null) url = url.concat("&namefilter=" + data.nameListFilter);
             if (data.followedOnly != null) url = url.concat("&followedonly=" + new Boolean(data.followedOnly).toString());
             if (data.excludeBlocked != null) url = url.concat("&excludeblocked=" + new Boolean(data.excludeBlocked).toString());
+            if (data.before != null) url = url.concat("&before=" + data.before);
+            if (data.after != null) url = url.concat("&after=" + data.after);
             if (data.limit != null) url = url.concat("&limit=" + data.limit);
             if (data.offset != null) url = url.concat("&offset=" + data.offset);
             if (data.reverse != null) url = url.concat("&reverse=" + new Boolean(data.reverse).toString());
@@ -448,6 +454,10 @@ function getDefaultTimeout(action) {
                 // User may take a long time to accept/deny the popup
                 return 60 * 60 * 1000;
 
+            case "SEARCH_QDN_RESOURCES":
+                // Searching for data can be slow, especially when metadata and statuses are also being included
+                return 30 * 1000;
+
             case "FETCH_QDN_RESOURCE":
                 // Fetching data can take a while, especially if the status hasn't been checked first
                 return 60 * 1000;

src/test/java/org/qortal/test/network/OnlineAccountsTests.java

@@ -51,89 +51,6 @@ public class OnlineAccountsTests extends Common {
     }
 
 
-    @Test
-    public void testGetOnlineAccountsV2() throws MessageException {
-        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(false);
-
-        Message messageOut = new GetOnlineAccountsV2Message(onlineAccountsOut);
-
-        byte[] messageBytes = messageOut.toBytes();
-        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
-
-        GetOnlineAccountsV2Message messageIn = (GetOnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
-
-        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
-
-        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
-        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
-
-        Message oldMessageOut = new GetOnlineAccountsMessage(onlineAccountsOut);
-        byte[] oldMessageBytes = oldMessageOut.toBytes();
-
-        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
-
-        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
-                onlineAccountsOut.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                oldMessageBytes.length,
-                messageBytes.length));
-    }
-
-    @Test
-    public void testOnlineAccountsV2() throws MessageException {
-        List<OnlineAccountData> onlineAccountsOut = generateOnlineAccounts(true);
-
-        Message messageOut = new OnlineAccountsV2Message(onlineAccountsOut);
-
-        byte[] messageBytes = messageOut.toBytes();
-        ByteBuffer byteBuffer = ByteBuffer.wrap(messageBytes);
-
-        OnlineAccountsV2Message messageIn = (OnlineAccountsV2Message) Message.fromByteBuffer(byteBuffer);
-
-        List<OnlineAccountData> onlineAccountsIn = messageIn.getOnlineAccounts();
-
-        assertEquals("size mismatch", onlineAccountsOut.size(), onlineAccountsIn.size());
-        assertTrue("accounts mismatch", onlineAccountsIn.containsAll(onlineAccountsOut));
-
-        Message oldMessageOut = new OnlineAccountsMessage(onlineAccountsOut);
-        byte[] oldMessageBytes = oldMessageOut.toBytes();
-
-        long numTimestamps = onlineAccountsOut.stream().mapToLong(OnlineAccountData::getTimestamp).sorted().distinct().count();
-
-        System.out.println(String.format("For %d accounts split across %d timestamp%s: old size %d vs new size %d",
-                onlineAccountsOut.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                oldMessageBytes.length,
-                messageBytes.length));
-    }
-
-    private List<OnlineAccountData> generateOnlineAccounts(boolean withSignatures) {
-        List<OnlineAccountData> onlineAccounts = new ArrayList<>();
-
-        int numTimestamps = RANDOM.nextInt(2) + 1; // 1 or 2
-
-        for (int t = 0; t < numTimestamps; ++t) {
-            int numAccounts = RANDOM.nextInt(3000);
-
-            for (int a = 0; a < numAccounts; ++a) {
-                byte[] sig = null;
-                if (withSignatures) {
-                    sig = new byte[Transformer.SIGNATURE_LENGTH];
-                    RANDOM.nextBytes(sig);
-                }
-
-                byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
-                RANDOM.nextBytes(pubkey);
-
-                onlineAccounts.add(new OnlineAccountData(t << 32, sig, pubkey));
-            }
-        }
-
-        return onlineAccounts;
-    }
-
     @Test
     public void testOnlineAccountsModulusV1() throws IllegalAccessException, DataException {
         try (final Repository repository = RepositoryManager.getRepository()) {

src/test/java/org/qortal/test/network/OnlineAccountsV3Tests.java

@@ -26,41 +26,6 @@ public class OnlineAccountsV3Tests {
         Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
     }
 
-    @Ignore("For informational use")
-    @Test
-    public void compareV2ToV3() throws MessageException {
-        List<OnlineAccountData> onlineAccounts = generateOnlineAccounts(false);
-
-        // How many of each timestamp and leading byte (of public key)
-        Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = convertToHashMaps(onlineAccounts);
-
-        byte[] v3DataBytes = new GetOnlineAccountsV3Message(hashesByTimestampThenByte).toBytes();
-        int v3ByteSize = v3DataBytes.length;
-
-        byte[] v2DataBytes = new GetOnlineAccountsV2Message(onlineAccounts).toBytes();
-        int v2ByteSize = v2DataBytes.length;
-
-        int numTimestamps = hashesByTimestampThenByte.size();
-        System.out.printf("For %d accounts split across %d timestamp%s: V2 size %d vs V3 size %d%n",
-                onlineAccounts.size(),
-                numTimestamps,
-                numTimestamps != 1 ? "s" : "",
-                v2ByteSize,
-                v3ByteSize
-        );
-
-        for (var outerMapEntry : hashesByTimestampThenByte.entrySet()) {
-            long timestamp = outerMapEntry.getKey();
-
-            var innerMap = outerMapEntry.getValue();
-
-            System.out.printf("For timestamp %d: %d / 256 slots used.%n",
-                    timestamp,
-                    innerMap.size()
-            );
-        }
-    }
-
     private Map<Long, Map<Byte, byte[]>> convertToHashMaps(List<OnlineAccountData> onlineAccounts) {
         // How many of each timestamp and leading byte (of public key)
         Map<Long, Map<Byte, byte[]>> hashesByTimestampThenByte = new HashMap<>();
@@ -200,7 +165,9 @@ public class OnlineAccountsV3Tests {
                 byte[] pubkey = new byte[Transformer.PUBLIC_KEY_LENGTH];
                 RANDOM.nextBytes(pubkey);
 
-                onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey));
+                Integer nonce = RANDOM.nextInt();
+
+                onlineAccounts.add(new OnlineAccountData(timestamp, sig, pubkey, nonce));
             }
         }