Added ArbitraryDataCache

This decides whether to build a new state or use an existing cached state when serving a data resource. It will cache a built resource until a new transaction (i.e. layer) arrives. This drastically reduces load, and still allows for almost instant propagation of new layers.

src/main/java/org/qortal/api/resource/WebsiteResource.java

@@ -293,12 +293,12 @@ public class WebsiteResource {
         ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service);
         arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
         try {
-            // TODO: overwrite if new transaction arrives, to invalidate cache
             // We could store the latest transaction signature in the extracted folder
             arbitraryDataReader.load(false);
         } catch (Exception e) {
             return this.get404Response();
         }
+
         java.nio.file.Path path = arbitraryDataReader.getFilePath();
         if (path == null) {
             return this.get404Response();

src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java

@@ -0,0 +1,158 @@
+package org.qortal.arbitrary;
+
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
+import org.qortal.controller.ArbitraryDataManager;
+import org.qortal.data.transaction.ArbitraryTransactionData;
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.repository.DataException;
+import org.qortal.repository.Repository;
+import org.qortal.repository.RepositoryManager;
+import org.qortal.utils.FilesystemUtils;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+
+public class ArbitraryDataCache {
+
+    private boolean overwrite;
+    private Path filePath;
+    private String resourceId;
+    private ResourceIdType resourceIdType;
+    private Service service;
+
+    public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId,
+                              ResourceIdType resourceIdType, Service service) {
+        this.filePath = filePath;
+        this.overwrite = overwrite;
+        this.resourceId = resourceId;
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+    }
+
+    public boolean shouldInvalidate() {
+        try {
+            // If the user has requested an overwrite, always invalidate the cache
+            if (this.overwrite) {
+                return true;
+            }
+
+            // Overwrite is false, but we still need to invalidate if no files exist
+            if (!Files.exists(this.filePath) || FilesystemUtils.isDirectoryEmpty(this.filePath)) {
+                return true;
+            }
+
+            // We might want to overwrite anyway, if an updated version is available
+            if (this.shouldInvalidateResource()) {
+                return true;
+            }
+
+        } catch (IOException e) {
+            // Something went wrong, so invalidate the cache just in case
+            return true;
+        }
+
+        // No need to invalidate the cache
+        return false;
+    }
+
+    private boolean shouldInvalidateResource() {
+        switch (this.resourceIdType) {
+
+            case NAME:
+                return this.shouldInvalidateName();
+
+            default:
+                // Other resource ID types remain constant, so no need to invalidate
+                return false;
+        }
+    }
+
+    private boolean shouldInvalidateName() {
+        // To avoid spamming the database too often, we shouldn't check sigs or invalidate when rate limited
+        if (this.rateLimitInEffect()) {
+            return false;
+        }
+
+        // If the state's sig doesn't match the latest transaction's sig, we need to invalidate
+        // This means that an updated layer is available
+        if (this.shouldInvalidateDueToSignatureMismatch()) {
+
+            // Add to the in-memory cache first, so that we won't check again for a while
+            ArbitraryDataManager.getInstance().addResourceToCache(this.resourceId);
+            return true;
+        }
+
+        return false;
+    }
+
+    /**
+     * rateLimitInEffect()
+     *
+     * When loading a website, we need to check the cache for every static asset loaded by the page.
+     * This would involve asking the database for the latest transaction every time.
+     * To reduce database load and page load times, we maintain an in-memory list to "rate limit" lookups.
+     * Once a resource ID is in this in-memory list, we will avoid cache invalidations until it
+     * has been present in the list for a certain amount of time.
+     * Items are automatically removed from the list when a new arbitrary transaction arrives, so this
+     * should not prevent updates from taking effect immediately.
+     *
+     * @return whether to avoid lookups for this resource due to the in-memory cache
+     */
+    private boolean rateLimitInEffect() {
+        return ArbitraryDataManager.getInstance().isResourceCached(this.resourceId);
+    }
+
+    private boolean shouldInvalidateDueToSignatureMismatch() {
+
+        // Fetch the latest transaction for this name and service
+        byte[] latestTransactionSig = this.fetchLatestTransactionSignature();
+
+        // Now fetch the transaction signature stored in the cache metadata
+        byte[] cachedSig = this.fetchCachedSignature();
+
+        // If either are null, we should invalidate
+        if (latestTransactionSig == null || cachedSig == null) {
+            return true;
+        }
+
+        // Check if they match
+        if (!Arrays.equals(latestTransactionSig, cachedSig)) {
+            return true;
+        }
+        return false;
+    }
+
+    private byte[] fetchLatestTransactionSignature() {
+        try (final Repository repository = RepositoryManager.getRepository()) {
+
+            // Find latest transaction for name and service, with any method
+            ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository()
+                    .getLatestTransaction(this.resourceId, this.service, Method.PUT);
+
+            if (latestTransaction != null) {
+                return latestTransaction.getSignature();
+            }
+
+        } catch (DataException e) {
+            return null;
+        }
+
+        return null;
+    }
+
+    private byte[] fetchCachedSignature() {
+        try {
+            // Fetch the transaction signature stored in the cache metadata
+            ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.filePath);
+            cache.read();
+            return cache.getSignature();
+
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -3,6 +3,7 @@ package org.qortal.arbitrary;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
 import org.qortal.crypto.AES;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -50,19 +51,23 @@ public class ArbitraryDataReader {
         this.resourceId = resourceId;
         this.resourceIdType = resourceIdType;
         this.service = service;
+
+        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
+        String baseDir = Settings.getInstance().getTempDataPath();
+        this.workingPath = Paths.get(baseDir, "reader", this.resourceId);
+        this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
     }
 
     public void load(boolean overwrite) throws IllegalStateException, IOException, DataException {
         try {
-            this.preExecute();
-
-            // Do nothing if files already exist and overwrite is set to false
-            if (!overwrite && Files.exists(this.uncompressedPath)
-                    && !FilesystemUtils.isDirectoryEmpty(this.uncompressedPath)) {
+            ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
+                    this.resourceId, this.resourceIdType, this.service);
+            if (!cache.shouldInvalidate()) {
                 this.filePath = this.uncompressedPath;
                 return;
             }
 
+            this.preExecute();
             this.deleteExistingFiles();
             this.fetch();
             this.decrypt();
@@ -83,19 +88,14 @@ public class ArbitraryDataReader {
     }
 
     private void createWorkingDirectory() {
-        // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
-        String baseDir = Settings.getInstance().getTempDataPath();
-        Path tempDir = Paths.get(baseDir, "reader", this.resourceId);
         try {
-            Files.createDirectories(tempDir);
+            Files.createDirectories(this.workingPath);
         } catch (IOException e) {
             throw new IllegalStateException("Unable to create temp directory");
         }
-        this.workingPath = tempDir;
     }
 
     private void createUncompressedDirectory() {
-        this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
         try {
             Files.createDirectories(this.uncompressedPath);
         } catch (IOException e) {

src/main/java/org/qortal/controller/ArbitraryDataManager.java

@@ -58,6 +58,19 @@ public class ArbitraryDataManager extends Thread {
 	 */
 	private Map<String, Long> arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>());
 
+	/**
+	 * Map to keep track of cached arbitrary transaction resources.
+	 * When an item is present in this list with a timestamp in the future, we won't invalidate
+	 * its cache when serving that data. This reduces the amount of database lookups that are needed.
+	 */
+	private Map<String, Long> arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>());
+
+	/**
+	 * The amount of time to cache a data resource before it is invalidated
+	 */
+	private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
+
+
 	private ArbitraryDataManager() {
 	}
 
@@ -197,11 +210,49 @@ public class ArbitraryDataManager extends Thread {
 
 	public void cleanupRequestCache(long now) {
 		final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT;
-		arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp);
+		arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // TODO: fix NPE
 		arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp);
 	}
 
 
+	// Arbitrary data resource cache
+	public boolean isResourceCached(String resourceId) {
+
+		// We don't have an entry for this resource ID, it is not cached
+		if (this.arbitraryDataCachedResources == null) {
+			return false;
+		}
+		if (!this.arbitraryDataCachedResources.containsKey(resourceId)) {
+			return false;
+		}
+		Long timestamp = this.arbitraryDataCachedResources.get(resourceId);
+		if (timestamp == null) {
+			return false;
+		}
+
+		// If the timestamp has reached the timeout, we should remove it from the cache
+		long now = NTP.getTime();
+		if (now > timestamp) {
+			this.arbitraryDataCachedResources.remove(resourceId);
+			return false;
+		}
+
+		// Current time hasn't reached the timeout, so treat it as cached
+		return true;
+	}
+
+	public void addResourceToCache(String resourceId) {
+		// Just in case
+		if (this.arbitraryDataCachedResources == null) {
+			this.arbitraryDataCachedResources = new HashMap<>();
+		}
+
+		// Set the timestamp to now + the timeout
+		Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT;
+		this.arbitraryDataCachedResources.put(resourceId, timestamp);
+	}
+
+
 	// Network handlers
 
 	public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) {
@@ -313,6 +364,17 @@ public class ArbitraryDataManager extends Thread {
 				}
 			}
 
+			// If we have all the chunks for this transaction's name, we should invalidate the data cache
+			// so that it is rebuilt the next time we serve it
+			if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(arbitraryTransactionData.getChunkHashes())) {
+				if (arbitraryTransactionData.getName() != null) {
+					String resourceId = arbitraryTransactionData.getName();
+					if (this.arbitraryDataCachedResources.containsKey(resourceId)) {
+						this.arbitraryDataCachedResources.remove(resourceId);
+					}
+				}
+			}
+
 		} catch (DataException | InterruptedException e) {
 			LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e);
 		}

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -258,15 +258,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 	@Override
 	public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException {
-		String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
+		StringBuilder sql = new StringBuilder(1024);
+
+		sql.append("SELECT type, reference, signature, creator, created_when, fee, " +
 				"tx_group_id, block_height, approval_status, approval_height, " +
 				"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
 				"name, update_method, secret, compression FROM ArbitraryTransactions " +
 				"JOIN Transactions USING (signature) " +
-				"WHERE name = ? AND service = ? AND update_method = ? " +
-				"ORDER BY created_when DESC LIMIT 1";
+				"WHERE name = ? AND service = ?");
 
-		try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) {
+		if (method != null) {
+			sql.append(" AND update_method = ");
+			sql.append(method.value);
+		}
+
+		sql.append("ORDER BY created_when DESC LIMIT 1");
+
+		try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name, service.value)) {
 			if (resultSet == null)
 				return null;