Initial attempt at a database cache to hold arbitrary resources and metadata.

src/main/java/org/qortal/api/resource/ArbitraryResource.java

@@ -86,12 +86,12 @@ public class ArbitraryResource {
 					"- If default is set to true, only resources without identifiers will be returned.",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> getResources(
+	public List<ArbitraryResourceData> getResources(
 			@QueryParam("service") Service service,
 			@QueryParam("name") String name,
 			@QueryParam("identifier") String identifier,
@@ -133,8 +133,9 @@ public class ArbitraryResource {
 				}
 			}
 
-			List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
+			List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
-					.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
+					.getArbitraryResources(service, identifier, names, defaultRes, followedOnly, excludeBlocked,
+							includeMetadata, limit, offset, reverse);
 
 			if (resources == null) {
 				return new ArrayList<>();
@@ -143,9 +144,6 @@ public class ArbitraryResource {
 			if (includeStatus != null && includeStatus) {
 				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
 			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
 
 			return resources;
 
@@ -161,12 +159,12 @@ public class ArbitraryResource {
 					"If default is set to true, only resources without identifiers will be returned.",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> searchResources(
+	public List<ArbitraryResourceData> searchResources(
 			@QueryParam("service") Service service,
 			@Parameter(description = "Query (searches both name and identifier fields)") @QueryParam("query") String query,
 			@Parameter(description = "Identifier (searches identifier field only)") @QueryParam("identifier") String identifier,
@@ -206,8 +204,9 @@ public class ArbitraryResource {
 				names = null;
 			}
 
-			List<ArbitraryResourceInfo> resources = repository.getArbitraryRepository()
+			List<ArbitraryResourceData> resources = repository.getArbitraryRepository()
-					.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames, defaultRes, followedOnly, excludeBlocked, limit, offset, reverse);
+					.searchArbitraryResources(service, query, identifier, names, usePrefixOnly, exactMatchNames,
+							defaultRes, followedOnly, excludeBlocked, includeMetadata, limit, offset, reverse);
 
 			if (resources == null) {
 				return new ArrayList<>();
@@ -216,9 +215,6 @@ public class ArbitraryResource {
 			if (includeStatus != null && includeStatus) {
 				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
 			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
 
 			return resources;
 
@@ -479,21 +475,20 @@ public class ArbitraryResource {
 			summary = "List arbitrary resources hosted by this node",
 			responses = {
 					@ApiResponse(
-							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceInfo.class))
+							content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceData.class))
 					)
 			}
 	)
 	@ApiErrors({ApiError.REPOSITORY_ISSUE})
-	public List<ArbitraryResourceInfo> getHostedResources(
+	public List<ArbitraryResourceData> getHostedResources(
 			@HeaderParam(Security.API_KEY_HEADER) String apiKey,
 			@Parameter(description = "Include status") @QueryParam("includestatus") Boolean includeStatus,
-			@Parameter(description = "Include metadata") @QueryParam("includemetadata") Boolean includeMetadata,
 			@Parameter(ref = "limit") @QueryParam("limit") Integer limit,
 			@Parameter(ref = "offset") @QueryParam("offset") Integer offset,
 			@QueryParam("query") String query) {
 		Security.checkApiCallAllowed(request);
 
-		List<ArbitraryResourceInfo> resources = new ArrayList<>();
+		List<ArbitraryResourceData> resources = new ArrayList<>();
 
 		try (final Repository repository = RepositoryManager.getRepository()) {
 			
@@ -509,21 +504,18 @@ public class ArbitraryResource {
 				if (transactionData.getService() == null) {
 					continue;
 				}
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
-				arbitraryResourceInfo.name = transactionData.getName();
+				arbitraryResourceData.name = transactionData.getName();
-				arbitraryResourceInfo.service = transactionData.getService();
+				arbitraryResourceData.service = transactionData.getService();
-				arbitraryResourceInfo.identifier = transactionData.getIdentifier();
+				arbitraryResourceData.identifier = transactionData.getIdentifier();
-				if (!resources.contains(arbitraryResourceInfo)) {
+				if (!resources.contains(arbitraryResourceData)) {
-					resources.add(arbitraryResourceInfo);
+					resources.add(arbitraryResourceData);
 				}
 			}
 
 			if (includeStatus != null && includeStatus) {
 				resources = ArbitraryTransactionUtils.addStatusToResources(resources);
 			}
-			if (includeMetadata != null && includeMetadata) {
-				resources = ArbitraryTransactionUtils.addMetadataToResources(resources);
-			}
 
 			return resources;
 

src/main/java/org/qortal/arbitrary/misc/Category.java

@@ -67,9 +67,12 @@ public enum Category {
     /**
      * Same as valueOf() but with fallback to UNCATEGORIZED if there's no match
      * @param name
-     * @return a Category (using UNCATEGORIZED if no match found)
+     * @return a Category (using UNCATEGORIZED if no match found), or null if null name passed
      */
     public static Category uncategorizedValueOf(String name) {
+        if (name == null) {
+            return null;
+        }
         try {
             return Category.valueOf(name);
         }

src/main/java/org/qortal/controller/Controller.java

@@ -47,6 +47,7 @@ import org.qortal.data.block.BlockData;
 import org.qortal.data.block.BlockSummaryData;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.network.PeerData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ChatTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.event.Event;
@@ -400,6 +401,10 @@ public class Controller extends Thread {
 			RepositoryFactory repositoryFactory = new HSQLDBRepositoryFactory(getRepositoryUrl());
 			RepositoryManager.setRepositoryFactory(repositoryFactory);
 			RepositoryManager.setRequestedCheckpoint(Boolean.TRUE);
+
+			try (final Repository repository = RepositoryManager.getRepository()) {
+				RepositoryManager.buildInitialArbitraryResourcesCache(repository);
+			}
 		}
 		catch (DataException e) {
 			// If exception has no cause then repository is in use by some other process.
@@ -891,6 +896,7 @@ public class Controller extends Thread {
 				if (now >= transaction.getDeadline()) {
 					LOGGER.debug(() -> String.format("Deleting expired, unconfirmed transaction %s", Base58.encode(transactionData.getSignature())));
 					repository.getTransactionRepository().delete(transactionData);
+					this.onExpiredTransaction(transactionData);
 					deletedCount++;
 				}
 			}
@@ -1203,6 +1209,21 @@ public class Controller extends Thread {
 		});
 	}
 
+	/**
+	 * Callback for when we've deleted an expired, unconfirmed transaction.
+	 * <p>
+	 * @implSpec performs actions in a new thread
+	 */
+	public void onExpiredTransaction(TransactionData transactionData) {
+		this.callbackExecutor.execute(() -> {
+
+			// If this is an ARBITRARY transaction, we may need to update the cache
+			if (transactionData.getType() == TransactionType.ARBITRARY) {
+				ArbitraryDataManager.getInstance().onExpiredArbitraryTransaction((ArbitraryTransactionData)transactionData);
+			}
+		});
+	}
+
 	public void onPeerHandshakeCompleted(Peer peer) {
 		// Only send if outbound
 		if (peer.isOutbound()) {

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -14,6 +14,7 @@ import org.qortal.arbitrary.ArbitraryDataResource;
 import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
 import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
@@ -539,6 +540,41 @@ public class ArbitraryDataManager extends Thread {
 		return true;
 	}
 
+	public void onExpiredArbitraryTransaction(ArbitraryTransactionData arbitraryTransactionData) {
+		if (arbitraryTransactionData.getName() == null) {
+			// No name, so we don't care about this transaction
+			return;
+		}
+
+		Service service = arbitraryTransactionData.getService();
+		String name = arbitraryTransactionData.getName();
+		String identifier = arbitraryTransactionData.getIdentifier();
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		try (final Repository repository = RepositoryManager.getRepository()) {
+			// Find next oldest transaction (which is now the latest transaction)
+			ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(name, service, null, identifier);
+
+			if (latestTransactionData == null) {
+				// There are no transactions anymore, so we can delete from the cache entirely (this deletes metadata too)
+				repository.getArbitraryRepository().delete(arbitraryResourceData);
+			}
+			else {
+				// We found the next oldest transaction, so we can update the cache
+				ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, latestTransactionData);
+				arbitraryTransaction.updateArbitraryResourceCache();
+				arbitraryTransaction.updateArbitraryMetadataCache();
+			}
+;
+		} catch (DataException  e) {
+			// Not much we can do, so ignore for now
+		}
+	}
+
 	public int getPowDifficulty() {
 		return this.powDifficulty;
 	}

src/main/java/org/qortal/controller/arbitrary/ArbitraryMetadataManager.java

@@ -15,6 +15,7 @@ import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
 import org.qortal.repository.RepositoryManager;
 import org.qortal.settings.Settings;
+import org.qortal.transaction.ArbitraryTransaction;
 import org.qortal.utils.Base58;
 import org.qortal.utils.ListUtils;
 import org.qortal.utils.NTP;
@@ -324,37 +325,44 @@ public class ArbitraryMetadataManager {
         Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
         arbitraryMetadataRequests.put(message.getId(), newEntry);
 
-        ArbitraryTransactionData arbitraryTransactionData = null;
+        // Get transaction info
-
+        try (final Repository repository = RepositoryManager.getRepository()) {
-        // Forwarding
+            TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-        if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
+            if (!(transactionData instanceof ArbitraryTransactionData)) {
-
+                return;
-            // Get transaction info
-            try (final Repository repository = RepositoryManager.getRepository()) {
-                TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
-                if (!(transactionData instanceof ArbitraryTransactionData))
-                    return;
-                arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
-            } catch (DataException e) {
-                LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
             }
+            ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-            // Check if the name is blocked
+            // Forwarding
-            boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+            if (isRelayRequest && Settings.getInstance().isRelayModeEnabled()) {
-            if (!isBlocked) {
-                Peer requestingPeer = request.getB();
-                if (requestingPeer != null) {
 
-                    ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
+                // Check if the name is blocked
-                    forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
+                boolean isBlocked = (arbitraryTransactionData == null || ListUtils.isNameBlocked(arbitraryTransactionData.getName()));
+                if (!isBlocked) {
+                    Peer requestingPeer = request.getB();
+                    if (requestingPeer != null) {
 
-                    // Forward to requesting peer
+                        ArbitraryMetadataMessage forwardArbitraryMetadataMessage = new ArbitraryMetadataMessage(signature, arbitraryMetadataMessage.getArbitraryMetadataFile());
-                    LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                        forwardArbitraryMetadataMessage.setId(arbitraryMetadataMessage.getId());
-                    if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+
-                        requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        // Forward to requesting peer
+                        LOGGER.debug("Forwarding metadata to requesting peer: {}", requestingPeer);
+                        if (!requestingPeer.sendMessage(forwardArbitraryMetadataMessage)) {
+                            requestingPeer.disconnect("failed to forward arbitrary metadata");
+                        }
                     }
                 }
             }
+
+            // Update arbitrary resource caches
+            if (arbitraryTransactionData != null) {
+                ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, arbitraryTransactionData);
+                arbitraryTransaction.updateArbitraryResourceCache();
+                arbitraryTransaction.updateArbitraryMetadataCache();
+            }
+
+        } catch (DataException e) {
+            LOGGER.error(String.format("Repository issue while finding arbitrary transaction metadata for peer %s", peer), e);
         }
     }
 

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceData.java

@@ -7,7 +7,7 @@ import javax.xml.bind.annotation.XmlAccessorType;
 import java.util.Objects;
 
 @XmlAccessorType(XmlAccessType.FIELD)
-public class ArbitraryResourceInfo {
+public class ArbitraryResourceData {
 
 	public String name;
 	public Service service;
@@ -15,11 +15,11 @@ public class ArbitraryResourceInfo {
 	public ArbitraryResourceStatus status;
 	public ArbitraryResourceMetadata metadata;
 
-	public Long size;
+	public Integer size;
 	public Long created;
 	public Long updated;
 
-	public ArbitraryResourceInfo() {
+	public ArbitraryResourceData() {
 	}
 
 	@Override
@@ -32,10 +32,10 @@ public class ArbitraryResourceInfo {
 		if (o == this)
 			return true;
 
-		if (!(o instanceof ArbitraryResourceInfo))
+		if (!(o instanceof ArbitraryResourceData))
 			return false;
 
-		ArbitraryResourceInfo other = (ArbitraryResourceInfo) o;
+		ArbitraryResourceData other = (ArbitraryResourceData) o;
 
 		return Objects.equals(this.name, other.name) &&
 				Objects.equals(this.service, other.service) &&

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceMetadata.java

@@ -18,6 +18,9 @@ public class ArbitraryResourceMetadata {
     private List<String> files;
     private String mimeType;
 
+    // Only included when updating database
+    private ArbitraryResourceData arbitraryResourceData;
+
     public ArbitraryResourceMetadata() {
     }
 
@@ -60,4 +63,48 @@ public class ArbitraryResourceMetadata {
     public List<String> getFiles() {
         return this.files;
     }
+
+    public void setTitle(String title) {
+        this.title = title;
+    }
+
+    public String getTitle() {
+        return this.title;
+    }
+
+    public void setDescription(String description) {
+        this.description = description;
+    }
+
+    public String getDescription() {
+        return this.description;
+    }
+
+    public void setTags(List<String> tags) {
+        this.tags = tags;
+    }
+
+    public List<String> getTags() {
+        return this.tags;
+    }
+
+    public void setCategory(Category category) {
+        this.category = category;
+
+        // Also set categoryName
+        if (category != null) {
+            this.categoryName = category.getName();
+        }
+    }
+
+    public Category getCategory() {
+        return this.category;
+    }
+
+    public void setArbitraryResourceData(ArbitraryResourceData arbitraryResourceData) {
+        this.arbitraryResourceData = arbitraryResourceData;
+    }
+    public ArbitraryResourceData getArbitraryResourceData() {
+        return this.arbitraryResourceData;
+    }
 }

src/main/java/org/qortal/data/arbitrary/ArbitraryResourceNameInfo.java

@@ -9,7 +9,7 @@ import java.util.List;
 public class ArbitraryResourceNameInfo {
 
 	public String name;
-	public List<ArbitraryResourceInfo> resources = new ArrayList<>();
+	public List<ArbitraryResourceData> resources = new ArrayList<>();
 
 	public ArbitraryResourceNameInfo() {
 	}

src/main/java/org/qortal/repository/ArbitraryRepository.java

@@ -1,9 +1,8 @@
 package org.qortal.repository;
 
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
-import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
-import org.qortal.data.network.ArbitraryPeerData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
 
@@ -11,23 +10,42 @@ import java.util.List;
 
 public interface ArbitraryRepository {
 
+	// Utils
+
 	public boolean isDataLocal(byte[] signature) throws DataException;
 
 	public byte[] fetchData(byte[] signature) throws DataException;
 
+
+	// Transaction related
+
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
 
 	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
 
 	public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, String identifier, long since) throws DataException;
 
+	public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException;
+
 	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException;
 
 
-	public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	// Resource related
 
-	public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException;
 
-	public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier, boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException;
+	public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException;
 
+	public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly, List<String> namesFilter, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked, Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException;
+
+
+	// Arbitrary resources cache save/load
+
+	public void save(ArbitraryResourceData arbitraryResourceData) throws DataException;
+	public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException;
+
+	public void save(ArbitraryResourceMetadata metadata) throws DataException;
+	public void delete(ArbitraryResourceMetadata metadata) throws DataException;
 }

src/main/java/org/qortal/repository/RepositoryManager.java

@@ -2,8 +2,16 @@ package org.qortal.repository;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.api.resource.TransactionsResource;
+import org.qortal.controller.Controller;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.settings.Settings;
+import org.qortal.transaction.ArbitraryTransaction;
+import org.qortal.transaction.Transaction;
 
 import java.sql.SQLException;
+import java.util.List;
 import java.util.concurrent.TimeoutException;
 
 public abstract class RepositoryManager {
@@ -56,6 +64,69 @@ public abstract class RepositoryManager {
 		}
 	}
 
+	public static boolean buildInitialArbitraryResourcesCache(Repository repository) throws DataException {
+		if (Settings.getInstance().isLite()) {
+			// Lite nodes have no blockchain
+			return false;
+		}
+
+		try {
+			// Check if QDNResources table is empty
+			List<ArbitraryResourceData> resources = repository.getArbitraryRepository().getArbitraryResources(10, 0, false);
+			if (!resources.isEmpty()) {
+				// Resources exist in the cache, so assume complete.
+				// We avoid checkpointing and prevent the node from starting up in the case of a rebuild failure, so
+				// we shouldn't ever be left in a partially rebuilt state.
+				LOGGER.debug("Arbitrary resources cache already built");
+				return false;
+			}
+
+			LOGGER.info("Building arbitrary resources cache...");
+
+			final int batchSize = 100;
+			int offset = 0;
+
+			// Loop through all ARBITRARY transactions, and determine latest state
+			while (!Controller.isStopping()) {
+				LOGGER.info("Fetching arbitrary transactions {} - {}", offset, offset+batchSize-1);
+
+				List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, List.of(Transaction.TransactionType.ARBITRARY), null, null, null, TransactionsResource.ConfirmationStatus.BOTH, batchSize, offset, false);
+				if (signatures.isEmpty()) {
+					// Complete
+					break;
+				}
+
+				// Expand signatures to transactions
+				for (byte[] signature : signatures) {
+					ArbitraryTransactionData transactionData = (ArbitraryTransactionData) repository
+							.getTransactionRepository().fromSignature(signature);
+
+					if (transactionData.getService() == null) {
+						// Unsupported service - ignore this resource
+						continue;
+					}
+
+					// Update arbitrary resource caches
+					ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
+					arbitraryTransaction.updateArbitraryResourceCache();
+					arbitraryTransaction.updateArbitraryMetadataCache();
+				}
+				offset += batchSize;
+			}
+
+			repository.saveChanges();
+			LOGGER.info("Completed build of initial arbitrary resources cache.");
+			return true;
+		}
+		catch (DataException e) {
+			LOGGER.info("Unable to build initial arbitrary resources cache: {}. The database may have been left in an inconsistent state.", e.getMessage());
+
+			// Throw an exception so that the node startup is halted, allowing for a retry next time.
+			repository.discardChanges();
+			throw new DataException("Build of initial arbitrary resources cache failed.");
+		}
+	}
+
 	public static void setRequestedCheckpoint(Boolean quick) {
 		quickCheckpointRequested = quick;
 	}

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -2,10 +2,10 @@ package org.qortal.repository.hsqldb;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.bouncycastle.util.Longs;
+import org.qortal.arbitrary.misc.Category;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
-import org.qortal.data.arbitrary.ArbitraryResourceNameInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
 import org.qortal.data.transaction.BaseTransactionData;
@@ -22,6 +22,7 @@ import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
@@ -41,6 +42,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return (ArbitraryTransactionData) transactionData;
 	}
 
+
+	// Utils
+
 	@Override
 	public boolean isDataLocal(byte[] signature) throws DataException {
 		ArbitraryTransactionData transactionData = getTransactionData(signature);
@@ -113,6 +117,9 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		return null;
 	}
 
+
+	// Transaction related
+
 	@Override
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
 		// Already hashed? Nothing to do
@@ -211,8 +218,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		}
 	}
 
-	@Override
+	private ArbitraryTransactionData getSingleTransaction(String name, Service service, Method method, String identifier, boolean firstNotLast) throws DataException {
-	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
 		StringBuilder sql = new StringBuilder(1024);
 
 		sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
@@ -228,7 +234,16 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			sql.append(method.value);
 		}
 
-		sql.append("ORDER BY created_when DESC LIMIT 1");
+		sql.append(" ORDER BY created_when");
+
+		if (firstNotLast) {
+			sql.append(" ASC");
+		}
+		else {
+			sql.append(" DESC");
+		}
+
+		sql.append(" LIMIT 1");
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name.toLowerCase(), service.value, identifier, identifier)) {
 			if (resultSet == null)
@@ -284,13 +299,189 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 	}
 
 	@Override
-	public List<ArbitraryResourceInfo> getArbitraryResources(Service service, String identifier, List<String> names,
+	public ArbitraryTransactionData getInitialTransaction(String name, Service service, Method method, String identifier) throws DataException {
-															 boolean defaultResource,  Boolean followedOnly, Boolean excludeBlocked,
+		return this.getSingleTransaction(name, service, method, identifier, true);
-															 Integer limit, Integer offset, Boolean reverse) throws DataException {
+	}
+
+	@Override
+	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method, String identifier) throws DataException {
+		return this.getSingleTransaction(name, service, method, identifier, false);
+	}
+
+
+	// Resource related
+
+	@Override
+	public ArbitraryResourceData getArbitraryResource(Service service, String name, String identifier) throws DataException {
 		StringBuilder sql = new StringBuilder(512);
 		List<Object> bindParams = new ArrayList<>();
 
-		sql.append("SELECT name, service, identifier, MAX(size) AS max_size FROM ArbitraryTransactions WHERE 1=1");
+		// Name is required
+		if (name == null) {
+			return null;
+		}
+
+		sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE service = ? AND name = ?");
+
+		bindParams.add(service.value);
+		bindParams.add(name);
+
+		if (identifier != null) {
+			sql.append(" AND identifier = ?");
+			bindParams.add(identifier);
+		}
+		else {
+			sql.append(" AND identifier IS NULL");
+		}
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return null;
+
+			String nameResult = resultSet.getString(1);
+			Service serviceResult = Service.valueOf(resultSet.getInt(2));
+			String identifierResult = resultSet.getString(3);
+			Integer sizeResult = resultSet.getInt(4);
+			Long created = resultSet.getLong(5);
+			Long updated = resultSet.getLong(6);
+
+			// Optional metadata fields
+			String title = resultSet.getString(7);
+			String description = resultSet.getString(8);
+			String category = resultSet.getString(9);
+			String tag1 = resultSet.getString(10);
+			String tag2 = resultSet.getString(11);
+			String tag3 = resultSet.getString(12);
+			String tag4 = resultSet.getString(13);
+			String tag5 = resultSet.getString(14);
+
+			if (Objects.equals(identifierResult, "default")) {
+				// Map "default" back to null. This is optional but probably less confusing than returning "default".
+				identifierResult = null;
+			}
+
+			ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+			arbitraryResourceData.name = nameResult;
+			arbitraryResourceData.service = serviceResult;
+			arbitraryResourceData.identifier = identifierResult;
+			arbitraryResourceData.size = sizeResult;
+			arbitraryResourceData.created = created;
+			arbitraryResourceData.updated = (updated == 0) ? null : updated;
+
+			ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+			metadata.setTitle(title);
+			metadata.setDescription(description);
+			metadata.setCategory(Category.uncategorizedValueOf(category));
+
+			List<String> tags = new ArrayList<>();
+			if (tag1 != null) tags.add(tag1);
+			if (tag2 != null) tags.add(tag2);
+			if (tag3 != null) tags.add(tag3);
+			if (tag4 != null) tags.add(tag4);
+			if (tag5 != null) tags.add(tag5);
+			metadata.setTags(!tags.isEmpty() ? tags : null);
+
+			arbitraryResourceData.metadata = metadata;
+
+			return arbitraryResourceData;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary resource from repository", e);
+		}
+	}
+	@Override
+	public List<ArbitraryResourceData> getArbitraryResources(Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE name IS NOT NULL ORDER BY created_when");
+
+		if (reverse != null && reverse) {
+			sql.append(" DESC");
+		}
+
+		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
+
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
+			if (resultSet == null)
+				return arbitraryResources;
+
+			do {
+				String nameResult = resultSet.getString(1);
+				Service serviceResult = Service.valueOf(resultSet.getInt(2));
+				String identifierResult = resultSet.getString(3);
+				Integer sizeResult = resultSet.getInt(4);
+				Long created = resultSet.getLong(5);
+				Long updated = resultSet.getLong(6);
+
+				// Optional metadata fields
+				String title = resultSet.getString(7);
+				String description = resultSet.getString(8);
+				String category = resultSet.getString(9);
+				String tag1 = resultSet.getString(10);
+				String tag2 = resultSet.getString(11);
+				String tag3 = resultSet.getString(12);
+				String tag4 = resultSet.getString(13);
+				String tag5 = resultSet.getString(14);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
+				}
+
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+				arbitraryResourceData.name = nameResult;
+				arbitraryResourceData.service = serviceResult;
+				arbitraryResourceData.identifier = identifierResult;
+				arbitraryResourceData.size = sizeResult;
+				arbitraryResourceData.created = created;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
+
+				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+				metadata.setTitle(title);
+				metadata.setDescription(description);
+				metadata.setCategory(Category.uncategorizedValueOf(category));
+
+				List<String> tags = new ArrayList<>();
+				if (tag1 != null) tags.add(tag1);
+				if (tag2 != null) tags.add(tag2);
+				if (tag3 != null) tags.add(tag3);
+				if (tag4 != null) tags.add(tag4);
+				if (tag5 != null) tags.add(tag5);
+				metadata.setTags(!tags.isEmpty() ? tags : null);
+
+				arbitraryResourceData.metadata = metadata;
+
+				arbitraryResources.add(arbitraryResourceData);
+			} while (resultSet.next());
+
+			return arbitraryResources;
+		} catch (SQLException e) {
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
+		}
+	}
+
+	@Override
+	public List<ArbitraryResourceData> getArbitraryResources(Service service, String identifier, List<String> names,
+															 boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
+															 Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		StringBuilder sql = new StringBuilder(512);
+		List<Object> bindParams = new ArrayList<>();
+
+		sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
+				"FROM ArbitraryResourcesCache " +
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE name IS NOT NULL");
 
 		if (service != null) {
 			sql.append(" AND service = ");
@@ -351,7 +542,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			}
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
+		sql.append(" ORDER BY created_when");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -359,49 +550,82 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
 
-		List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
-				return null;
+				return arbitraryResources;
 
 			do {
 				String nameResult = resultSet.getString(1);
 				Service serviceResult = Service.valueOf(resultSet.getInt(2));
 				String identifierResult = resultSet.getString(3);
 				Integer sizeResult = resultSet.getInt(4);
+				Long created = resultSet.getLong(5);
+				Long updated = resultSet.getLong(6);
 
-				// We should filter out resources without names
+				// Optional metadata fields
-				if (nameResult == null) {
+				String title = resultSet.getString(7);
-					continue;
+				String description = resultSet.getString(8);
+				String category = resultSet.getString(9);
+				String tag1 = resultSet.getString(10);
+				String tag2 = resultSet.getString(11);
+				String tag3 = resultSet.getString(12);
+				String tag4 = resultSet.getString(13);
+				String tag5 = resultSet.getString(14);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
 				}
 
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
-				arbitraryResourceInfo.name = nameResult;
+				arbitraryResourceData.name = nameResult;
-				arbitraryResourceInfo.service = serviceResult;
+				arbitraryResourceData.service = serviceResult;
-				arbitraryResourceInfo.identifier = identifierResult;
+				arbitraryResourceData.identifier = identifierResult;
-				arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
+				arbitraryResourceData.size = sizeResult;
+				arbitraryResourceData.created = created;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
 
-				arbitraryResources.add(arbitraryResourceInfo);
+				if (includeMetadata != null && includeMetadata) {
+					// TODO: we could avoid the join altogether
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setTitle(title);
+					metadata.setDescription(description);
+					metadata.setCategory(Category.uncategorizedValueOf(category));
+
+					List<String> tags = new ArrayList<>();
+					if (tag1 != null) tags.add(tag1);
+					if (tag2 != null) tags.add(tag2);
+					if (tag3 != null) tags.add(tag3);
+					if (tag4 != null) tags.add(tag4);
+					if (tag5 != null) tags.add(tag5);
+					metadata.setTags(!tags.isEmpty() ? tags : null);
+
+					arbitraryResourceData.metadata = metadata;
+				}
+
+				arbitraryResources.add(arbitraryResourceData);
 			} while (resultSet.next());
 
 			return arbitraryResources;
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
 		}
 	}
 
 	@Override
-	public List<ArbitraryResourceInfo> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
+	public List<ArbitraryResourceData> searchArbitraryResources(Service service, String query, String identifier, List<String> names, boolean prefixOnly,
 																List<String> exactMatchNames, boolean defaultResource, Boolean followedOnly, Boolean excludeBlocked,
-																Integer limit, Integer offset, Boolean reverse) throws DataException {
+																Boolean includeMetadata, Integer limit, Integer offset, Boolean reverse) throws DataException {
 		StringBuilder sql = new StringBuilder(512);
 		List<Object> bindParams = new ArrayList<>();
 
-		sql.append("SELECT name, service, identifier, MAX(size) AS max_size, MIN(created_when) AS date_created, MAX(created_when) AS date_updated " +
+		sql.append("SELECT name, service, identifier, size, created_when, updated_when, " +
-				"FROM ArbitraryTransactions " +
+				"title, description, category, tag1, tag2, tag3, tag4, tag5 " +
-				"JOIN Transactions USING (signature) " +
+				"FROM ArbitraryResourcesCache " +
-				"WHERE 1=1");
+				"LEFT JOIN ArbitraryMetadataCache USING (service, name, identifier) " +
+				"WHERE name IS NOT NULL");
 
 		if (service != null) {
 			sql.append(" AND service = ");
@@ -492,7 +716,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 			}
 		}
 
-		sql.append(" GROUP BY name, service, identifier ORDER BY date_created");
+		sql.append(" ORDER BY created_when");
 
 		if (reverse != null && reverse) {
 			sql.append(" DESC");
@@ -500,98 +724,156 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
 
-		List<ArbitraryResourceInfo> arbitraryResources = new ArrayList<>();
+		List<ArbitraryResourceData> arbitraryResources = new ArrayList<>();
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), bindParams.toArray())) {
 			if (resultSet == null)
-				return null;
+				return arbitraryResources;
 
 			do {
 				String nameResult = resultSet.getString(1);
 				Service serviceResult = Service.valueOf(resultSet.getInt(2));
 				String identifierResult = resultSet.getString(3);
 				Integer sizeResult = resultSet.getInt(4);
-				long dateCreated = resultSet.getLong(5);
+				Long created = resultSet.getLong(5);
-				long dateUpdated = resultSet.getLong(6);
+				Long updated = resultSet.getLong(6);
 
-				// We should filter out resources without names
+				// Optional metadata fields
-				if (nameResult == null) {
+				String title = resultSet.getString(7);
-					continue;
+				String description = resultSet.getString(8);
+				String category = resultSet.getString(9);
+				String tag1 = resultSet.getString(10);
+				String tag2 = resultSet.getString(11);
+				String tag3 = resultSet.getString(12);
+				String tag4 = resultSet.getString(13);
+				String tag5 = resultSet.getString(14);
+
+				if (Objects.equals(identifierResult, "default")) {
+					// Map "default" back to null. This is optional but probably less confusing than returning "default".
+					identifierResult = null;
 				}
 
-				ArbitraryResourceInfo arbitraryResourceInfo = new ArbitraryResourceInfo();
+				ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
-				arbitraryResourceInfo.name = nameResult;
+				arbitraryResourceData.name = nameResult;
-				arbitraryResourceInfo.service = serviceResult;
+				arbitraryResourceData.service = serviceResult;
-				arbitraryResourceInfo.identifier = identifierResult;
+				arbitraryResourceData.identifier = identifierResult;
-				arbitraryResourceInfo.size = Longs.valueOf(sizeResult);
+				arbitraryResourceData.size = sizeResult;
-				arbitraryResourceInfo.created = dateCreated;
+				arbitraryResourceData.created = created;
-				arbitraryResourceInfo.updated = dateUpdated;
+				arbitraryResourceData.updated = (updated == 0) ? null : updated;
 
-				arbitraryResources.add(arbitraryResourceInfo);
+				if (includeMetadata != null && includeMetadata) {
+					// TODO: we could avoid the join altogether
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setTitle(title);
+					metadata.setDescription(description);
+					metadata.setCategory(Category.uncategorizedValueOf(category));
+
+					List<String> tags = new ArrayList<>();
+					if (tag1 != null) tags.add(tag1);
+					if (tag2 != null) tags.add(tag2);
+					if (tag3 != null) tags.add(tag3);
+					if (tag4 != null) tags.add(tag4);
+					if (tag5 != null) tags.add(tag5);
+					metadata.setTags(!tags.isEmpty() ? tags : null);
+
+					arbitraryResourceData.metadata = metadata;
+				}
+
+				arbitraryResources.add(arbitraryResourceData);
 			} while (resultSet.next());
 
 			return arbitraryResources;
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to fetch arbitrary resources from repository", e);
+		}
+	}
+
+
+	// Arbitrary resources cache save/load
+
+	@Override
+	public void save(ArbitraryResourceData arbitraryResourceData) throws DataException {
+		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryResourcesCache");
+
+		saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
+				.bind("identifier", arbitraryResourceData.identifier).bind("size", arbitraryResourceData.size)
+				.bind("created_when", arbitraryResourceData.created).bind("updated_when", arbitraryResourceData.updated);
+
+		try {
+			saveHelper.execute(this.repository);
+		} catch (SQLException e) {
+			throw new DataException("Unable to save arbitrary resource info into repository", e);
 		}
 	}
 
 	@Override
-	public List<ArbitraryResourceNameInfo> getArbitraryResourceCreatorNames(Service service, String identifier,
+	public void delete(ArbitraryResourceData arbitraryResourceData) throws DataException {
-																			boolean defaultResource, Integer limit, Integer offset, Boolean reverse) throws DataException {
+		// NOTE: arbitrary metadata are deleted automatically by the database thanks to "ON DELETE CASCADE"
-		StringBuilder sql = new StringBuilder(512);
+		// in ArbitraryMetadataCache' FOREIGN KEY definition.
+		try {
+			this.repository.delete("ArbitraryResourcesCache", "service = ? AND name = ? AND identifier = ?",
+					arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
 
-		sql.append("SELECT name FROM ArbitraryTransactions WHERE 1=1");
-
-		if (service != null) {
-			sql.append(" AND service = ");
-			sql.append(service.value);
-		}
-
-		if (defaultResource) {
-			// Default resource requested - use NULL identifier
-			// The AND ? IS NULL AND ? IS NULL is a hack to make use of the identifier params in checkedExecute()
-			identifier = null;
-			sql.append(" AND (identifier IS NULL AND ? IS NULL AND ? IS NULL)");
-		}
-		else {
-			// Non-default resource requested
-			// Use an exact match identifier, or list all if supplied identifier is null
-			sql.append(" AND (identifier = ? OR (? IS NULL))");
-		}
-
-		sql.append(" GROUP BY name ORDER BY name COLLATE SQL_TEXT_UCC_NO_PAD");
-
-		if (reverse != null && reverse) {
-			sql.append(" DESC");
-		}
-
-		HSQLDBRepository.limitOffsetSql(sql, limit, offset);
-
-		List<ArbitraryResourceNameInfo> arbitraryResources = new ArrayList<>();
-
-		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), identifier, identifier)) {
-			if (resultSet == null)
-				return null;
-
-			do {
-				String name = resultSet.getString(1);
-
-				// We should filter out resources without names
-				if (name == null) {
-					continue;
-				}
-
-				ArbitraryResourceNameInfo arbitraryResourceNameInfo = new ArbitraryResourceNameInfo();
-				arbitraryResourceNameInfo.name = name;
-
-				arbitraryResources.add(arbitraryResourceNameInfo);
-			} while (resultSet.next());
-
-			return arbitraryResources;
 		} catch (SQLException e) {
-			throw new DataException("Unable to fetch arbitrary transactions from repository", e);
+			throw new DataException("Unable to delete account from repository", e);
 		}
 	}
 
+
+	/* Arbitrary metadata cache */
+
+	@Override
+	public void save(ArbitraryResourceMetadata metadata) throws DataException {
+		HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryMetadataCache");
+
+		ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
+		if (arbitraryResourceData == null) {
+			throw new DataException("Can't save metadata without a referenced resource");
+		}
+
+		String tag1 = null;
+		String tag2 = null;
+		String tag3 = null;
+		String tag4 = null;
+		String tag5 = null;
+
+		List<String> tags = metadata.getTags();
+		if (tags != null) {
+			if (tags.size() > 0) tag1 = tags.get(0);
+			if (tags.size() > 1) tag2 = tags.get(1);
+			if (tags.size() > 2) tag3 = tags.get(2);
+			if (tags.size() > 3) tag4 = tags.get(3);
+			if (tags.size() > 4) tag5 = tags.get(4);
+		}
+
+		String category = metadata.getCategory() != null ? metadata.getCategory().toString() : null;
+
+		saveHelper.bind("service", arbitraryResourceData.service.value).bind("name", arbitraryResourceData.name)
+				.bind("identifier", arbitraryResourceData.identifier).bind("title", metadata.getTitle())
+				.bind("description", metadata.getDescription()).bind("category", category)
+				.bind("tag1", tag1).bind("tag2", tag2).bind("tag3", tag3).bind("tag4", tag4)
+				.bind("tag5", tag5);
+
+		try {
+			saveHelper.execute(this.repository);
+		} catch (SQLException e) {
+			throw new DataException("Unable to save arbitrary metadata into repository", e);
+		}
+	}
+
+	@Override
+	public void delete(ArbitraryResourceMetadata metadata) throws DataException {
+		ArbitraryResourceData arbitraryResourceData = metadata.getArbitraryResourceData();
+		if (arbitraryResourceData == null) {
+			throw new DataException("Can't delete metadata without a referenced resource");
+		}
+
+		try {
+			this.repository.delete("ArbitraryMetadataCache", "service = ? AND name = ? AND identifier = ?",
+					arbitraryResourceData.service.value, arbitraryResourceData.name, arbitraryResourceData.identifier);
+
+		} catch (SQLException e) {
+			throw new DataException("Unable to delete account from repository", e);
+		}
+	}
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -901,7 +901,7 @@ public class HSQLDBDatabaseUpdates {
 				case 37:
 					// ARBITRARY transaction updates for off-chain data storage
 
-					// We may want to use a nonce rather than a transaction fee on the data chain
+					// We may want to use a nonce rather than a transaction fee for ARBITRARY transactions
 					stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0");
 					// We need to know the total size of the data file(s) associated with each transaction
 					stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0");
@@ -993,6 +993,44 @@ public class HSQLDBDatabaseUpdates {
 					stmt.execute("ALTER TABLE CancelSellNameTransactions ADD sale_price QortalAmount");
 					break;
 
+				case 47:
+					// We need to keep a local cache of arbitrary resources (items published to QDN), for easier searching.
+					// IMPORTANT: this is a cache of the last known state of a resource (both confirmed
+					// and valid unconfirmed). It cannot be assumed that all nodes will contain the same state at a
+					// given block height, and therefore must NOT be used for any consensus/validation code. It is
+					// simply a cache, to avoid having to query the raw transactions and the metadata in flat files
+					// when serving API requests.
+					// ARBITRARY transactions aren't really suitable for updating resources in the same way we'd update
+					// names or groups for instance, as there is no distinction between creations and updates, and metadata
+					// is off-chain. Plus, QDN allows (valid) unconfirmed data to be queried and viewed. It is very
+					// easy to keep a cache of the latest transaction's data, but anything more than that would need
+					// considerable thought (and most likely a rewrite).
+
+					stmt.execute("CREATE TABLE ArbitraryResourcesCache (service SMALLINT NOT NULL, "
+							+ "name RegisteredName NOT NULL, identifier VARCHAR(64), size INT NOT NULL, "
+							+ "created_when EpochMillis NOT NULL, updated_when EpochMillis, "
+							+ "PRIMARY KEY (service, name, identifier))");
+					// For finding resources by service.
+					stmt.execute("CREATE INDEX ArbitraryResourcesServiceIndex ON ArbitraryResourcesCache (service)");
+					// For finding resources by name.
+					stmt.execute("CREATE INDEX ArbitraryResourcesNameIndex ON ArbitraryResourcesCache (name)");
+					// For finding resources by identifier.
+					stmt.execute("CREATE INDEX ArbitraryResourcesIdentifierIndex ON ArbitraryResourcesCache (identifier)");
+					// For finding resources by creation date (the default column when ordering).
+					stmt.execute("CREATE INDEX ArbitraryResourcesCreatedIndex ON ArbitraryResourcesCache (created_when)");
+					// Use a separate table space as this table will be very large.
+					stmt.execute("SET TABLE ArbitraryResourcesCache NEW SPACE");
+
+					stmt.execute("CREATE TABLE ArbitraryMetadataCache (service SMALLINT NOT NULL, "
+							+ "name RegisteredName NOT NULL, identifier VARCHAR(64), "
+							+ "title VARCHAR(80), description VARCHAR(240), category VARCHAR(64), "
+							+ "tag1 VARCHAR(20), tag2 VARCHAR(20), tag3 VARCHAR(20), tag4 VARCHAR(20), tag5 VARCHAR(20), "
+							+ "PRIMARY KEY (service, name, identifier), FOREIGN KEY (service, name, identifier) "
+							+ "REFERENCES ArbitraryResourcesCache (service, name, identifier) ON DELETE CASCADE)");
+					// For finding metadata by title.
+					stmt.execute("CREATE INDEX ArbitraryMetadataTitleIndex ON ArbitraryMetadataCache (title)");
+					break;
+
 				default:
 					// nothing to do
 					return false;

src/main/java/org/qortal/transaction/ArbitraryTransaction.java

@@ -1,18 +1,21 @@
 package org.qortal.transaction;
 
-import java.util.Arrays;
+import java.io.IOException;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
 import org.qortal.account.Account;
+import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
+import org.qortal.arbitrary.misc.Service;
 import org.qortal.block.BlockChain;
 import org.qortal.controller.arbitrary.ArbitraryDataManager;
-import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
 import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
 import org.qortal.crypto.Crypto;
 import org.qortal.crypto.MemoryPoW;
 import org.qortal.data.PaymentData;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
+import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.naming.NameData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -248,6 +251,10 @@ public class ArbitraryTransaction extends Transaction {
 				ArbitraryDataManager.getInstance().invalidateCache(arbitraryTransactionData);
 			}
 		}
+
+		// Add to arbitrary resource caches
+		this.updateArbitraryResourceCache();
+		this.updateArbitraryMetadataCache();
 	}
 
 	@Override
@@ -304,4 +311,123 @@ public class ArbitraryTransaction extends Transaction {
 		return null;
 	}
 
+	/**
+	 * Update the arbitrary resources cache.
+	 * This finds the latest transaction and replaces the
+	 * majority of the data in the cache. The current
+	 * transaction is used for the created time,
+	 * if it has a lower timestamp than the existing value.
+	 * It's also used to identify the correct
+	 * service/name/identifier combination.
+	 *
+	 * @throws DataException
+	 */
+	public void updateArbitraryResourceCache() throws DataException {
+		// Don't cache resources without a name (such as auto updates)
+		if (arbitraryTransactionData.getName() == null) {
+			return;
+		}
+
+		// Get the latest transaction
+		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
+		if (latestTransactionData == null) {
+			// We don't have a latest transaction, so give up
+			return;
+		}
+
+		Service service = arbitraryTransactionData.getService();
+		String name = arbitraryTransactionData.getName();
+		String identifier = arbitraryTransactionData.getIdentifier();
+
+		// In the cache we store null identifiers as "default", as it is part of the primary key
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		// Get existing cached entry if it exists
+		ArbitraryResourceData existingArbitraryResourceData = repository.getArbitraryRepository()
+				.getArbitraryResource(service, name, identifier);
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		// Check for existing cached data
+		if (existingArbitraryResourceData == null) {
+			// Nothing exists yet, so set everything from the newest transaction
+			arbitraryResourceData.created = latestTransactionData.getTimestamp();
+			arbitraryResourceData.updated = null;
+		}
+		else {
+			// An entry already exists - update created time from current transaction if this is older
+			arbitraryResourceData.created = Math.min(existingArbitraryResourceData.created, arbitraryTransactionData.getTimestamp());
+
+			// Set updated time to the latest transaction's timestamp, unless it matches the creation time
+			if (existingArbitraryResourceData.created == latestTransactionData.getTimestamp()) {
+				// Latest transaction matches created time, so it hasn't been updated
+				arbitraryResourceData.updated = null;
+			}
+			else {
+				arbitraryResourceData.updated = latestTransactionData.getTimestamp();
+			}
+		}
+
+		arbitraryResourceData.size = latestTransactionData.getSize();
+
+		// Save
+		repository.getArbitraryRepository().save(arbitraryResourceData);
+	}
+
+	public void updateArbitraryMetadataCache() throws DataException {
+		// Get the latest transaction
+		ArbitraryTransactionData latestTransactionData = repository.getArbitraryRepository().getLatestTransaction(arbitraryTransactionData.getName(), arbitraryTransactionData.getService(), null, arbitraryTransactionData.getIdentifier());
+		if (latestTransactionData == null) {
+			// We don't have a latest transaction, so give up
+			return;
+		}
+
+		Service service = latestTransactionData.getService();
+		String name = latestTransactionData.getName();
+		String identifier = latestTransactionData.getIdentifier();
+
+		// In the cache we store null identifiers as "default", as it is part of the primary key
+		if (identifier == null) {
+			identifier = "default";
+		}
+
+		ArbitraryResourceData arbitraryResourceData = new ArbitraryResourceData();
+		arbitraryResourceData.service = service;
+		arbitraryResourceData.name = name;
+		arbitraryResourceData.identifier = identifier;
+
+		// Update metadata for latest transaction if it is local
+		if (latestTransactionData.getMetadataHash() != null) {
+			ArbitraryDataFile metadataFile = ArbitraryDataFile.fromHash(latestTransactionData.getMetadataHash(), latestTransactionData.getSignature());
+			if (metadataFile.exists()) {
+				ArbitraryDataTransactionMetadata transactionMetadata = new ArbitraryDataTransactionMetadata(metadataFile.getFilePath());
+				try {
+					transactionMetadata.read();
+
+					ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+					metadata.setArbitraryResourceData(arbitraryResourceData);
+					metadata.setTitle(transactionMetadata.getTitle());
+					metadata.setDescription(transactionMetadata.getDescription());
+					metadata.setCategory(transactionMetadata.getCategory());
+					metadata.setTags(transactionMetadata.getTags());
+					repository.getArbitraryRepository().save(metadata);
+
+				} catch (IOException e) {
+					// Ignore, as we can add it again later
+				}
+			} else {
+				// We don't have a local copy of this metadata file, so delete it from the cache
+				// It will be re-added if the file later arrives via the network
+				ArbitraryResourceMetadata metadata = new ArbitraryResourceMetadata();
+				metadata.setArbitraryResourceData(arbitraryResourceData);
+				repository.getArbitraryRepository().delete(metadata);
+			}
+		}
+	}
+
 }

src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java

@@ -4,10 +4,8 @@ import org.apache.commons.lang3.ArrayUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.arbitrary.*;
-import org.qortal.arbitrary.metadata.ArbitraryDataTransactionMetadata;
 import org.qortal.arbitrary.misc.Service;
-import org.qortal.data.arbitrary.ArbitraryResourceInfo;
+import org.qortal.data.arbitrary.ArbitraryResourceData;
-import org.qortal.data.arbitrary.ArbitraryResourceMetadata;
 import org.qortal.data.arbitrary.ArbitraryResourceStatus;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
@@ -258,8 +256,7 @@ public class ArbitraryTransactionUtils {
                                 "chunks if needed", Base58.encode(completeHash));
 
                         ArbitraryTransactionUtils.deleteCompleteFile(arbitraryTransactionData, now, cleanupAfter);
-                    }
+                    } else {
-                    else {
                         // File might be in use. It's best to leave it and it it will be cleaned up later.
                     }
                 }
@@ -271,6 +268,7 @@ public class ArbitraryTransactionUtils {
      * When first uploaded, files go into a _misc folder as they are not yet associated with a
      * transaction signature. Once the transaction is broadcast, they need to be moved to the
      * correct location, keyed by the transaction signature.
+     *
      * @param arbitraryTransactionData
      * @return
      * @throws DataException
@@ -356,8 +354,7 @@ public class ArbitraryTransactionUtils {
                     file.createNewFile();
                 }
             }
-        }
+        } catch (DataException | IOException e) {
-        catch (DataException | IOException e) {
             LOGGER.info("Unable to check and relocate all files for signature {}: {}",
                     Base58.encode(arbitraryTransactionData.getSignature()), e.getMessage());
         }
@@ -366,7 +363,7 @@ public class ArbitraryTransactionUtils {
     }
 
     public static List<ArbitraryTransactionData> limitOffsetTransactions(List<ArbitraryTransactionData> transactions,
-                                                                              Integer limit, Integer offset) {
+                                                                         Integer limit, Integer offset) {
         if (limit != null && limit == 0) {
             limit = null;
         }
@@ -389,6 +386,7 @@ public class ArbitraryTransactionUtils {
 
     /**
      * Lookup status of resource
+     *
      * @param service
      * @param name
      * @param identifier
@@ -413,10 +411,10 @@ public class ArbitraryTransactionUtils {
         return resource.getStatus(false);
     }
 
-    public static List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
+    public static List<ArbitraryResourceData> addStatusToResources(List<ArbitraryResourceData> resources) {
         // Determine and add the status of each resource
-        List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
+        List<ArbitraryResourceData> updatedResources = new ArrayList<>();
-        for (ArbitraryResourceInfo resourceInfo : resources) {
+        for (ArbitraryResourceData resourceInfo : resources) {
             try {
                 ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
                         resourceInfo.service, resourceInfo.identifier);
@@ -433,21 +431,4 @@ public class ArbitraryTransactionUtils {
         }
         return updatedResources;
     }
-
-    public static List<ArbitraryResourceInfo> addMetadataToResources(List<ArbitraryResourceInfo> resources) {
-        // Add metadata fields to each resource if they exist
-        List<ArbitraryResourceInfo> updatedResources = new ArrayList<>();
-        for (ArbitraryResourceInfo resourceInfo : resources) {
-            ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ArbitraryDataFile.ResourceIdType.NAME,
-                    resourceInfo.service, resourceInfo.identifier);
-            ArbitraryDataTransactionMetadata transactionMetadata = resource.getLatestTransactionMetadata();
-            ArbitraryResourceMetadata resourceMetadata = ArbitraryResourceMetadata.fromTransactionMetadata(transactionMetadata, false);
-            if (resourceMetadata != null) {
-                resourceInfo.metadata = resourceMetadata;
-            }
-            updatedResources.add(resourceInfo);
-        }
-        return updatedResources;
-    }
-
 }