diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index d9538cc1..45b8097e 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -293,12 +293,12 @@ public class WebsiteResource { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { - // TODO: overwrite if new transaction arrives, to invalidate cache // We could store the latest transaction signature in the extracted folder arbitraryDataReader.load(false); } catch (Exception e) { return this.get404Response(); } + java.nio.file.Path path = arbitraryDataReader.getFilePath(); if (path == null) { return this.get404Response(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java new file mode 100644 index 00000000..1cb45610 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -0,0 +1,158 @@ +package org.qortal.arbitrary; + +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; +import org.qortal.controller.ArbitraryDataManager; +import org.qortal.data.transaction.ArbitraryTransactionData; +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.utils.FilesystemUtils; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; + +public class ArbitraryDataCache { + + private boolean overwrite; + private Path filePath; + private String resourceId; + private ResourceIdType resourceIdType; + private Service service; + + public ArbitraryDataCache(Path filePath, boolean overwrite, String resourceId, + ResourceIdType resourceIdType, Service service) { + this.filePath = filePath; + this.overwrite = overwrite; + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + } + + public boolean shouldInvalidate() { + try { + // If the user has requested an overwrite, always invalidate the cache + if (this.overwrite) { + return true; + } + + // Overwrite is false, but we still need to invalidate if no files exist + if (!Files.exists(this.filePath) || FilesystemUtils.isDirectoryEmpty(this.filePath)) { + return true; + } + + // We might want to overwrite anyway, if an updated version is available + if (this.shouldInvalidateResource()) { + return true; + } + + } catch (IOException e) { + // Something went wrong, so invalidate the cache just in case + return true; + } + + // No need to invalidate the cache + return false; + } + + private boolean shouldInvalidateResource() { + switch (this.resourceIdType) { + + case NAME: + return this.shouldInvalidateName(); + + default: + // Other resource ID types remain constant, so no need to invalidate + return false; + } + } + + private boolean shouldInvalidateName() { + // To avoid spamming the database too often, we shouldn't check sigs or invalidate when rate limited + if (this.rateLimitInEffect()) { + return false; + } + + // If the state's sig doesn't match the latest transaction's sig, we need to invalidate + // This means that an updated layer is available + if (this.shouldInvalidateDueToSignatureMismatch()) { + + // Add to the in-memory cache first, so that we won't check again for a while + ArbitraryDataManager.getInstance().addResourceToCache(this.resourceId); + return true; + } + + return false; + } + + /** + * rateLimitInEffect() + * + * When loading a website, we need to check the cache for every static asset loaded by the page. + * This would involve asking the database for the latest transaction every time. + * To reduce database load and page load times, we maintain an in-memory list to "rate limit" lookups. + * Once a resource ID is in this in-memory list, we will avoid cache invalidations until it + * has been present in the list for a certain amount of time. + * Items are automatically removed from the list when a new arbitrary transaction arrives, so this + * should not prevent updates from taking effect immediately. + * + * @return whether to avoid lookups for this resource due to the in-memory cache + */ + private boolean rateLimitInEffect() { + return ArbitraryDataManager.getInstance().isResourceCached(this.resourceId); + } + + private boolean shouldInvalidateDueToSignatureMismatch() { + + // Fetch the latest transaction for this name and service + byte[] latestTransactionSig = this.fetchLatestTransactionSignature(); + + // Now fetch the transaction signature stored in the cache metadata + byte[] cachedSig = this.fetchCachedSignature(); + + // If either are null, we should invalidate + if (latestTransactionSig == null || cachedSig == null) { + return true; + } + + // Check if they match + if (!Arrays.equals(latestTransactionSig, cachedSig)) { + return true; + } + return false; + } + + private byte[] fetchLatestTransactionSignature() { + try (final Repository repository = RepositoryManager.getRepository()) { + + // Find latest transaction for name and service, with any method + ArbitraryTransactionData latestTransaction = repository.getArbitraryRepository() + .getLatestTransaction(this.resourceId, this.service, Method.PUT); + + if (latestTransaction != null) { + return latestTransaction.getSignature(); + } + + } catch (DataException e) { + return null; + } + + return null; + } + + private byte[] fetchCachedSignature() { + try { + // Fetch the transaction signature stored in the cache metadata + ArbitraryDataMetadataCache cache = new ArbitraryDataMetadataCache(this.filePath); + cache.read(); + return cache.getSignature(); + + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index a15c7f0e..deb17cf7 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -50,19 +51,23 @@ public class ArbitraryDataReader { this.resourceId = resourceId; this.resourceIdType = resourceIdType; this.service = service; + + // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware + String baseDir = Settings.getInstance().getTempDataPath(); + this.workingPath = Paths.get(baseDir, "reader", this.resourceId); + this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); } public void load(boolean overwrite) throws IllegalStateException, IOException, DataException { try { - this.preExecute(); - - // Do nothing if files already exist and overwrite is set to false - if (!overwrite && Files.exists(this.uncompressedPath) - && !FilesystemUtils.isDirectoryEmpty(this.uncompressedPath)) { + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, + this.resourceId, this.resourceIdType, this.service); + if (!cache.shouldInvalidate()) { this.filePath = this.uncompressedPath; return; } + this.preExecute(); this.deleteExistingFiles(); this.fetch(); this.decrypt(); @@ -83,19 +88,14 @@ public class ArbitraryDataReader { } private void createWorkingDirectory() { - // Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware - String baseDir = Settings.getInstance().getTempDataPath(); - Path tempDir = Paths.get(baseDir, "reader", this.resourceId); try { - Files.createDirectories(tempDir); + Files.createDirectories(this.workingPath); } catch (IOException e) { throw new IllegalStateException("Unable to create temp directory"); } - this.workingPath = tempDir; } private void createUncompressedDirectory() { - this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); try { Files.createDirectories(this.uncompressedPath); } catch (IOException e) { diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/ArbitraryDataManager.java index 10c5d448..df0c670d 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/ArbitraryDataManager.java @@ -58,6 +58,19 @@ public class ArbitraryDataManager extends Thread { */ private Map arbitraryDataFileRequests = Collections.synchronizedMap(new HashMap<>()); + /** + * Map to keep track of cached arbitrary transaction resources. + * When an item is present in this list with a timestamp in the future, we won't invalidate + * its cache when serving that data. This reduces the amount of database lookups that are needed. + */ + private Map arbitraryDataCachedResources = Collections.synchronizedMap(new HashMap<>()); + + /** + * The amount of time to cache a data resource before it is invalidated + */ + private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes + + private ArbitraryDataManager() { } @@ -197,11 +210,49 @@ public class ArbitraryDataManager extends Thread { public void cleanupRequestCache(long now) { final long requestMinimumTimestamp = now - ARBITRARY_REQUEST_TIMEOUT; - arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); + arbitraryDataFileListRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); // TODO: fix NPE arbitraryDataFileRequests.entrySet().removeIf(entry -> entry.getValue() < requestMinimumTimestamp); } + // Arbitrary data resource cache + public boolean isResourceCached(String resourceId) { + + // We don't have an entry for this resource ID, it is not cached + if (this.arbitraryDataCachedResources == null) { + return false; + } + if (!this.arbitraryDataCachedResources.containsKey(resourceId)) { + return false; + } + Long timestamp = this.arbitraryDataCachedResources.get(resourceId); + if (timestamp == null) { + return false; + } + + // If the timestamp has reached the timeout, we should remove it from the cache + long now = NTP.getTime(); + if (now > timestamp) { + this.arbitraryDataCachedResources.remove(resourceId); + return false; + } + + // Current time hasn't reached the timeout, so treat it as cached + return true; + } + + public void addResourceToCache(String resourceId) { + // Just in case + if (this.arbitraryDataCachedResources == null) { + this.arbitraryDataCachedResources = new HashMap<>(); + } + + // Set the timestamp to now + the timeout + Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT; + this.arbitraryDataCachedResources.put(resourceId, timestamp); + } + + // Network handlers public void onNetworkGetArbitraryDataMessage(Peer peer, Message message) { @@ -313,6 +364,17 @@ public class ArbitraryDataManager extends Thread { } } + // If we have all the chunks for this transaction's name, we should invalidate the data cache + // so that it is rebuilt the next time we serve it + if (arbitraryDataFile.exists() || arbitraryDataFile.allChunksExist(arbitraryTransactionData.getChunkHashes())) { + if (arbitraryTransactionData.getName() != null) { + String resourceId = arbitraryTransactionData.getName(); + if (this.arbitraryDataCachedResources.containsKey(resourceId)) { + this.arbitraryDataCachedResources.remove(resourceId); + } + } + } + } catch (DataException | InterruptedException e) { LOGGER.error(String.format("Repository issue while finding arbitrary transaction data list for peer %s", peer), e); } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java index b6a920af..16841cc1 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -258,15 +258,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException { - String sql = "SELECT type, reference, signature, creator, created_when, fee, " + + StringBuilder sql = new StringBuilder(1024); + + sql.append("SELECT type, reference, signature, creator, created_when, fee, " + "tx_group_id, block_height, approval_status, approval_height, " + "version, nonce, service, size, is_data_raw, data, chunk_hashes, " + "name, update_method, secret, compression FROM ArbitraryTransactions " + "JOIN Transactions USING (signature) " + - "WHERE name = ? AND service = ? AND update_method = ? " + - "ORDER BY created_when DESC LIMIT 1"; + "WHERE name = ? AND service = ?"); - try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) { + if (method != null) { + sql.append(" AND update_method = "); + sql.append(method.value); + } + + sql.append("ORDER BY created_when DESC LIMIT 1"); + + try (ResultSet resultSet = this.repository.checkedExecute(sql.toString(), name, service.value)) { if (resultSet == null) return null;