Initial implementation of arbitrary data build queue

This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method.

When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed.

src/main/java/org/qortal/api/resource/WebsiteResource.java

@@ -7,6 +7,8 @@ import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import java.io.*;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.NoSuchFileException;
 import java.nio.file.Paths;
@@ -14,6 +16,7 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
+import com.google.common.io.Resources;
 import io.swagger.v3.oas.annotations.Operation;
 import io.swagger.v3.oas.annotations.media.Content;
 import io.swagger.v3.oas.annotations.media.Schema;
@@ -293,8 +296,11 @@ public class WebsiteResource {
         ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service);
         arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
         try {
-            // We could store the latest transaction signature in the extracted folder
+            if (!arbitraryDataReader.isCachedDataAvailable()) {
-            arbitraryDataReader.load(false);
+                arbitraryDataReader.loadAsynchronously();
+                return this.getLoadingResponse();
+            }
+
         } catch (Exception e) {
             LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage()));
             return this.getResponse(500, "Error 500: Internal Server Error");
@@ -365,6 +371,17 @@ public class WebsiteResource {
         return userPath;
     }
 
+    private HttpServletResponse getLoadingResponse() {
+        String responseString = null;
+        URL url = Resources.getResource("loading/index.html");
+        try {
+            responseString = Resources.toString(url, StandardCharsets.UTF_8);
+        } catch (IOException e) {
+            LOGGER.info("Unable to show loading screen: {}", e.getMessage());
+        }
+        return this.getResponse(503, responseString);
+    }
+
     private HttpServletResponse getResponse(int responseCode, String responseString) {
         try {
             byte[] responseData = responseString.getBytes();

src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java

@@ -0,0 +1,69 @@
+package org.qortal.arbitrary;
+
+import org.qortal.data.transaction.ArbitraryTransactionData.*;
+import org.qortal.arbitrary.ArbitraryDataFile.*;
+import org.qortal.repository.DataException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+
+public class ArbitraryDataBuildQueueItem {
+
+    private String resourceId;
+    private ResourceIdType resourceIdType;
+    private Service service;
+    private Long buildStartTimestamp = null;
+
+    private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
+
+    public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) {
+        this.resourceId = resourceId;
+        this.resourceIdType = resourceIdType;
+        this.service = service;
+    }
+
+    public void build() throws IOException, DataException {
+        Long now = NTP.getTime();
+        if (now == null) {
+            throw new IllegalStateException("NTP time hasn't synced yet");
+        }
+
+        this.buildStartTimestamp = now;
+        ArbitraryDataReader arbitraryDataReader =
+                new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service);
+
+        // We do not want to overwrite the existing cache, as this will be invalidated
+        // automatically if new data has arrived
+        arbitraryDataReader.loadSynchronously(false);
+    }
+
+    public boolean isBuilding() {
+        return this.buildStartTimestamp != null;
+    }
+
+    public boolean isQueued() {
+        return this.buildStartTimestamp == null;
+    }
+
+    public boolean hasReachedBuildTimeout(Long now) {
+        if (now == null || this.buildStartTimestamp == null) {
+            return true;
+        }
+        return now - this.buildStartTimestamp > BUILD_TIMEOUT;
+    }
+
+
+    public String getResourceId() {
+        return this.resourceId;
+    }
+
+    public Long getBuildStartTimestamp() {
+        return this.buildStartTimestamp;
+    }
+
+    @Override
+    public String toString() {
+        return String.format("%s %s", this.service, this.resourceId);
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -112,7 +112,7 @@ public class ArbitraryDataBuilder {
             String sig58 = Base58.encode(transactionData.getSignature());
             ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
             arbitraryDataReader.setTransactionData(transactionData);
-            arbitraryDataReader.load(true);
+            arbitraryDataReader.loadSynchronously(true);
             Path path = arbitraryDataReader.getFilePath();
             if (path == null) {
                 throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58));

src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java

@@ -2,7 +2,7 @@ package org.qortal.arbitrary;
 
 import org.qortal.arbitrary.ArbitraryDataFile.*;
 import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
-import org.qortal.controller.ArbitraryDataManager;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
 import org.qortal.repository.DataException;

src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java

@@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.crypto.AES;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.*;
@@ -59,7 +60,54 @@ public class ArbitraryDataReader {
         this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
     }
 
-    public void load(boolean overwrite) throws IllegalStateException, IOException, DataException {
+    public boolean isCachedDataAvailable() {
+        // If this resource is in the build queue then we shouldn't attempt to serve
+        // cached data, as it may not be fully built
+        ArbitraryDataBuildQueueItem queueItem =
+                new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
+        if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) {
+            return false;
+        }
+
+        // Not in the build queue - so check the cache itself
+        ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
+                this.resourceId, this.resourceIdType, this.service);
+        if (!cache.shouldInvalidate()) {
+            this.filePath = this.uncompressedPath;
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * loadAsynchronously
+     *
+     * Attempts to load the resource asynchronously
+     * This adds the build task to a queue, and the result will be cached when complete
+     * To check the status of the build, periodically call isCachedDataAvailable()
+     * Once it returns true, you can then use getFilePath() to access the data itself.
+     * TODO: create API to check the status
+     * @return
+     */
+    public boolean loadAsynchronously() {
+        ArbitraryDataBuildQueueItem queueItem =
+                new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
+        return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem);
+    }
+
+    /**
+     * loadSynchronously
+     *
+     * Attempts to load the resource synchronously
+     * Warning: this can block for a long time when building or fetching complex data
+     * If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
+     *
+     * @param overwrite - set to true to force rebuild an existing cache
+     * @throws IllegalStateException
+     * @throws IOException
+     * @throws DataException
+     */
+    public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException {
         try {
             ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
                     this.resourceId, this.resourceIdType, this.service);

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java

@@ -45,7 +45,7 @@ public class ArbitraryDataMetadataCache extends ArbitraryDataMetadata {
         patch.put("timestamp", this.timestamp);
 
         this.jsonString = patch.toString(2);
-        LOGGER.info("Cache metadata: {}", this.jsonString);
+        LOGGER.trace("Cache metadata: {}", this.jsonString);
     }
 
 

src/main/java/org/qortal/controller/Controller.java

@@ -14,6 +14,7 @@ import org.qortal.block.Block;
 import org.qortal.block.BlockChain;
 import org.qortal.block.BlockChain.BlockTimingByHeight;
 import org.qortal.controller.Synchronizer.SynchronizationResult;
+import org.qortal.controller.arbitrary.ArbitraryDataManager;
 import org.qortal.controller.tradebot.TradeBot;
 import org.qortal.data.account.MintingAccountData;
 import org.qortal.data.account.RewardShareData;
@@ -332,7 +333,7 @@ public class Controller extends Thread {
 		return this.savedArgs;
 	}
 
-	/* package */ static boolean isStopping() {
+	/* package */ public static boolean isStopping() {
 		return isStopping;
 	}
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java

@@ -0,0 +1,78 @@
+package org.qortal.controller.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
+import org.qortal.controller.Controller;
+import org.qortal.repository.DataException;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.util.Map;
+
+
+public class ArbitraryDataBuildManager implements Runnable {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class);
+
+    public ArbitraryDataBuildManager() {
+
+    }
+
+    public void run() {
+        Thread.currentThread().setName("Arbitrary Data Build Manager");
+        ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance();
+
+        while (!Controller.isStopping()) {
+            try {
+                Thread.sleep(1000);
+
+                if (arbitraryDataManager.arbitraryDataBuildQueue == null) {
+                    continue;
+                }
+                if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) {
+                    continue;
+                }
+
+                // Find resources that are queued for building
+                Map.Entry<String, ArbitraryDataBuildQueueItem> next = arbitraryDataManager.arbitraryDataBuildQueue
+                        .entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get();
+
+                if (next == null) {
+                    continue;
+                }
+
+                Long now = NTP.getTime();
+                if (now == null) {
+                    continue;
+                }
+
+                String resourceId = next.getKey();
+                ArbitraryDataBuildQueueItem queueItem = next.getValue();
+                if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) {
+                    this.removeFromQueue(resourceId);
+                }
+
+                try {
+                    // Perform the build
+                    LOGGER.info("Building {}...", queueItem);
+                    queueItem.build();
+                    this.removeFromQueue(resourceId);
+                    LOGGER.info("Finished building {}", queueItem);
+
+                } catch (IOException | DataException e) {
+                    // Something went wrong - so remove it from the queue
+                    // TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts
+                    this.removeFromQueue(resourceId);
+                }
+
+            } catch (InterruptedException e) {
+                // Time to exit
+            }
+        }
+    }
+
+    private void removeFromQueue(String resourceId) {
+        ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId);
+    }
+}

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java

@@ -1,10 +1,14 @@
-package org.qortal.controller;
+package org.qortal.controller.arbitrary;
 
 import java.util.*;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
+import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
+import org.qortal.controller.Controller;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.network.Network;
@@ -70,6 +74,11 @@ public class ArbitraryDataManager extends Thread {
 	 */
 	private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
 
+	/**
+	 * Map to keep track of arbitrary transaction resources currently being built (or queued).
+	 */
+	public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
+
 
 	private ArbitraryDataManager() {
 	}
@@ -85,6 +94,11 @@ public class ArbitraryDataManager extends Thread {
 	public void run() {
 		Thread.currentThread().setName("Arbitrary Data Manager");
 
+		// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
+		// This can be expanded to have multiple threads processing the build queue when needed
+		ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
+		arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager());
+
 		try {
 			while (!isStopping) {
 				Thread.sleep(2000);
@@ -247,11 +261,62 @@ public class ArbitraryDataManager extends Thread {
 			this.arbitraryDataCachedResources = new HashMap<>();
 		}
 
+		Long now = NTP.getTime();
+		if (now == null) {
+			return;
+		}
+
 		// Set the timestamp to now + the timeout
 		Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT;
 		this.arbitraryDataCachedResources.put(resourceId, timestamp);
 	}
 
+	// Build queue
+	public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+		String resourceId = queueItem.getResourceId();
+		if (resourceId == null) {
+			return false;
+		}
+
+		if (this.arbitraryDataBuildQueue == null) {
+			return false;
+		}
+
+		if (NTP.getTime() == null) {
+			// Can't use queues until we have synced the time
+			return false;
+		}
+
+		if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) {
+			// Already in queue
+			return true;
+		}
+
+		LOGGER.info("Added {} to build queue", resourceId);
+
+		// Added to queue
+		return true;
+	}
+
+	public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
+		String resourceId = queueItem.getResourceId();
+		if (resourceId == null) {
+			return false;
+		}
+
+		if (this.arbitraryDataBuildQueue == null) {
+			return false;
+		}
+
+		if (this.arbitraryDataBuildQueue.containsKey(resourceId)) {
+			// Already in queue
+			return true;
+		}
+
+		// Not in queue
+		return false;
+	}
+
 
 	// Network handlers
 

src/main/resources/loading/index.html

@@ -0,0 +1,23 @@
+<html>
+<head>
+    <title>Loading...</title>
+    <script>
+        setTimeout(window.location.reload.bind(window.location), 1);
+    </script>
+    <style>
+        body {
+          font-family: Arial;
+          text-align: center;
+        }
+        h1 {
+            margin-top: 50px;
+        }
+    </style>
+</head>
+</html>
+<body>
+    <h1>Loading... please wait...</h1>
+    <p>This page will refresh automatically when the content becomes available</p>
+    <p>(We can show a Qortal branded loading screen here)</p>
+</body>
+</html>