From 79bbadad2f18bdfd090fe3496ac4739904198729 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 18 Aug 2021 07:50:45 +0100 Subject: [PATCH] Initial implementation of arbitrary data build queue This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method. When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed. --- .../qortal/api/resource/WebsiteResource.java | 21 ++++- .../ArbitraryDataBuildQueueItem.java | 69 ++++++++++++++++ .../arbitrary/ArbitraryDataBuilder.java | 2 +- .../qortal/arbitrary/ArbitraryDataCache.java | 2 +- .../qortal/arbitrary/ArbitraryDataReader.java | 50 +++++++++++- .../metadata/ArbitraryDataMetadataCache.java | 2 +- .../org/qortal/controller/Controller.java | 3 +- .../arbitrary/ArbitraryDataBuildManager.java | 78 +++++++++++++++++++ .../{ => arbitrary}/ArbitraryDataManager.java | 67 +++++++++++++++- src/main/resources/loading/index.html | 23 ++++++ 10 files changed, 309 insertions(+), 8 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java create mode 100644 src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java rename src/main/java/org/qortal/controller/{ => arbitrary}/ArbitraryDataManager.java (90%) create mode 100644 src/main/resources/loading/index.html diff --git a/src/main/java/org/qortal/api/resource/WebsiteResource.java b/src/main/java/org/qortal/api/resource/WebsiteResource.java index ea1fc6c5..87d439c5 100644 --- a/src/main/java/org/qortal/api/resource/WebsiteResource.java +++ b/src/main/java/org/qortal/api/resource/WebsiteResource.java @@ -7,6 +7,8 @@ import javax.ws.rs.*; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; import java.io.*; +import java.net.URL; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Paths; @@ -14,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import com.google.common.io.Resources; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.media.Schema; @@ -293,8 +296,11 @@ public class WebsiteResource { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service); arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only try { - // We could store the latest transaction signature in the extracted folder - arbitraryDataReader.load(false); + if (!arbitraryDataReader.isCachedDataAvailable()) { + arbitraryDataReader.loadAsynchronously(); + return this.getLoadingResponse(); + } + } catch (Exception e) { LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage())); return this.getResponse(500, "Error 500: Internal Server Error"); @@ -365,6 +371,17 @@ public class WebsiteResource { return userPath; } + private HttpServletResponse getLoadingResponse() { + String responseString = null; + URL url = Resources.getResource("loading/index.html"); + try { + responseString = Resources.toString(url, StandardCharsets.UTF_8); + } catch (IOException e) { + LOGGER.info("Unable to show loading screen: {}", e.getMessage()); + } + return this.getResponse(503, responseString); + } + private HttpServletResponse getResponse(int responseCode, String responseString) { try { byte[] responseData = responseString.getBytes(); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java new file mode 100644 index 00000000..2cabd3ec --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java @@ -0,0 +1,69 @@ +package org.qortal.arbitrary; + +import org.qortal.data.transaction.ArbitraryTransactionData.*; +import org.qortal.arbitrary.ArbitraryDataFile.*; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; + +public class ArbitraryDataBuildQueueItem { + + private String resourceId; + private ResourceIdType resourceIdType; + private Service service; + private Long buildStartTimestamp = null; + + private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds + + public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) { + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + } + + public void build() throws IOException, DataException { + Long now = NTP.getTime(); + if (now == null) { + throw new IllegalStateException("NTP time hasn't synced yet"); + } + + this.buildStartTimestamp = now; + ArbitraryDataReader arbitraryDataReader = + new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service); + + // We do not want to overwrite the existing cache, as this will be invalidated + // automatically if new data has arrived + arbitraryDataReader.loadSynchronously(false); + } + + public boolean isBuilding() { + return this.buildStartTimestamp != null; + } + + public boolean isQueued() { + return this.buildStartTimestamp == null; + } + + public boolean hasReachedBuildTimeout(Long now) { + if (now == null || this.buildStartTimestamp == null) { + return true; + } + return now - this.buildStartTimestamp > BUILD_TIMEOUT; + } + + + public String getResourceId() { + return this.resourceId; + } + + public Long getBuildStartTimestamp() { + return this.buildStartTimestamp; + } + + @Override + public String toString() { + return String.format("%s %s", this.service, this.resourceId); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index c8da187f..87795a18 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -112,7 +112,7 @@ public class ArbitraryDataBuilder { String sig58 = Base58.encode(transactionData.getSignature()); ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service); arbitraryDataReader.setTransactionData(transactionData); - arbitraryDataReader.load(true); + arbitraryDataReader.loadSynchronously(true); Path path = arbitraryDataReader.getFilePath(); if (path == null) { throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58)); diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java index 1cb45610..e15bbf37 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java @@ -2,7 +2,7 @@ package org.qortal.arbitrary; import org.qortal.arbitrary.ArbitraryDataFile.*; import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache; -import org.qortal.controller.ArbitraryDataManager; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.repository.DataException; diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 148ace8e..d35fa322 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.crypto.AES; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.ArbitraryTransactionData.*; @@ -59,7 +60,54 @@ public class ArbitraryDataReader { this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); } - public void load(boolean overwrite) throws IllegalStateException, IOException, DataException { + public boolean isCachedDataAvailable() { + // If this resource is in the build queue then we shouldn't attempt to serve + // cached data, as it may not be fully built + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); + if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) { + return false; + } + + // Not in the build queue - so check the cache itself + ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false, + this.resourceId, this.resourceIdType, this.service); + if (!cache.shouldInvalidate()) { + this.filePath = this.uncompressedPath; + return true; + } + return false; + } + + /** + * loadAsynchronously + * + * Attempts to load the resource asynchronously + * This adds the build task to a queue, and the result will be cached when complete + * To check the status of the build, periodically call isCachedDataAvailable() + * Once it returns true, you can then use getFilePath() to access the data itself. + * TODO: create API to check the status + * @return + */ + public boolean loadAsynchronously() { + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service); + return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem); + } + + /** + * loadSynchronously + * + * Attempts to load the resource synchronously + * Warning: this can block for a long time when building or fetching complex data + * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning + * + * @param overwrite - set to true to force rebuild an existing cache + * @throws IllegalStateException + * @throws IOException + * @throws DataException + */ + public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException { try { ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite, this.resourceId, this.resourceIdType, this.service); diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java index d6d7a2b4..bedbddee 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java @@ -45,7 +45,7 @@ public class ArbitraryDataMetadataCache extends ArbitraryDataMetadata { patch.put("timestamp", this.timestamp); this.jsonString = patch.toString(2); - LOGGER.info("Cache metadata: {}", this.jsonString); + LOGGER.trace("Cache metadata: {}", this.jsonString); } diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 44eb7f92..9d0914b7 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -14,6 +14,7 @@ import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.BlockTimingByHeight; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.tradebot.TradeBot; import org.qortal.data.account.MintingAccountData; import org.qortal.data.account.RewardShareData; @@ -332,7 +333,7 @@ public class Controller extends Thread { return this.savedArgs; } - /* package */ static boolean isStopping() { + /* package */ public static boolean isStopping() { return isStopping; } diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java new file mode 100644 index 00000000..a5325c03 --- /dev/null +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java @@ -0,0 +1,78 @@ +package org.qortal.controller.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.controller.Controller; +import org.qortal.repository.DataException; +import org.qortal.utils.NTP; + +import java.io.IOException; +import java.util.Map; + + +public class ArbitraryDataBuildManager implements Runnable { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class); + + public ArbitraryDataBuildManager() { + + } + + public void run() { + Thread.currentThread().setName("Arbitrary Data Build Manager"); + ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance(); + + while (!Controller.isStopping()) { + try { + Thread.sleep(1000); + + if (arbitraryDataManager.arbitraryDataBuildQueue == null) { + continue; + } + if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) { + continue; + } + + // Find resources that are queued for building + Map.Entry next = arbitraryDataManager.arbitraryDataBuildQueue + .entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get(); + + if (next == null) { + continue; + } + + Long now = NTP.getTime(); + if (now == null) { + continue; + } + + String resourceId = next.getKey(); + ArbitraryDataBuildQueueItem queueItem = next.getValue(); + if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) { + this.removeFromQueue(resourceId); + } + + try { + // Perform the build + LOGGER.info("Building {}...", queueItem); + queueItem.build(); + this.removeFromQueue(resourceId); + LOGGER.info("Finished building {}", queueItem); + + } catch (IOException | DataException e) { + // Something went wrong - so remove it from the queue + // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts + this.removeFromQueue(resourceId); + } + + } catch (InterruptedException e) { + // Time to exit + } + } + } + + private void removeFromQueue(String resourceId) { + ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId); + } +} diff --git a/src/main/java/org/qortal/controller/ArbitraryDataManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java similarity index 90% rename from src/main/java/org/qortal/controller/ArbitraryDataManager.java rename to src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java index df0c670d..4a99f55e 100644 --- a/src/main/java/org/qortal/controller/ArbitraryDataManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java @@ -1,10 +1,14 @@ -package org.qortal.controller; +package org.qortal.controller.arbitrary; import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; +import org.qortal.arbitrary.ArbitraryDataBuildQueueItem; +import org.qortal.controller.Controller; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.network.Network; @@ -70,6 +74,11 @@ public class ArbitraryDataManager extends Thread { */ private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes + /** + * Map to keep track of arbitrary transaction resources currently being built (or queued). + */ + public Map arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>()); + private ArbitraryDataManager() { } @@ -85,6 +94,11 @@ public class ArbitraryDataManager extends Thread { public void run() { Thread.currentThread().setName("Arbitrary Data Manager"); + // Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread) + // This can be expanded to have multiple threads processing the build queue when needed + ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1); + arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager()); + try { while (!isStopping) { Thread.sleep(2000); @@ -247,11 +261,62 @@ public class ArbitraryDataManager extends Thread { this.arbitraryDataCachedResources = new HashMap<>(); } + Long now = NTP.getTime(); + if (now == null) { + return; + } + // Set the timestamp to now + the timeout Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT; this.arbitraryDataCachedResources.put(resourceId, timestamp); } + // Build queue + public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (NTP.getTime() == null) { + // Can't use queues until we have synced the time + return false; + } + + if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) { + // Already in queue + return true; + } + + LOGGER.info("Added {} to build queue", resourceId); + + // Added to queue + return true; + } + + public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) { + String resourceId = queueItem.getResourceId(); + if (resourceId == null) { + return false; + } + + if (this.arbitraryDataBuildQueue == null) { + return false; + } + + if (this.arbitraryDataBuildQueue.containsKey(resourceId)) { + // Already in queue + return true; + } + + // Not in queue + return false; + } + // Network handlers diff --git a/src/main/resources/loading/index.html b/src/main/resources/loading/index.html new file mode 100644 index 00000000..eaf7ee1b --- /dev/null +++ b/src/main/resources/loading/index.html @@ -0,0 +1,23 @@ + + + Loading... + + + + + +

Loading... please wait...

+

This page will refresh automatically when the content becomes available

+

(We can show a Qortal branded loading screen here)

+ +