AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Initial implementation of arbitrary data build queue

Browse Source 
This adds the loadAsynchronously() method to ArbitraryDataReader, in addition to the existing loadSynchronously() method.

When requesting a website in a browser, previously the building of the resource's layers would be done synchronously in the API handler. This understandably caused many issues, so the building is now done asynchronously by a dedicated thread. A loading screen is shown in its place which auto refreshes every second until the build has completed.
This commit is contained in:
CalDescent 2021-08-18 07:50:45 +01:00
parent c3b44cee94
commit 79bbadad2f
10 changed files with 309 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@ -7,6 +7,8 @@ import javax.ws.rs.*;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.io.*;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Paths;
@ -14,6 +16,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import com.google.common.io.Resources;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.media.Content;
import io.swagger.v3.oas.annotations.media.Schema;
@ -293,8 +296,11 @@ public class WebsiteResource {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(resourceId, resourceIdType, service);
arbitraryDataReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try {
// We could store the latest transaction signature in the extracted folder
arbitraryDataReader.load(false);
if (!arbitraryDataReader.isCachedDataAvailable()) {
arbitraryDataReader.loadAsynchronously();
return this.getLoadingResponse();
}
} catch (Exception e) {
LOGGER.info(String.format("Unable to load %s %s: %s", service, resourceId, e.getMessage()));
return this.getResponse(500, "Error 500: Internal Server Error");
@ -365,6 +371,17 @@ public class WebsiteResource {
return userPath;
}
private HttpServletResponse getLoadingResponse() {
String responseString = null;
URL url = Resources.getResource("loading/index.html");
try {
responseString = Resources.toString(url, StandardCharsets.UTF_8);
} catch (IOException e) {
LOGGER.info("Unable to show loading screen: {}", e.getMessage());
}
return this.getResponse(503, responseString);
}
private HttpServletResponse getResponse(int responseCode, String responseString) {
try {
byte[] responseData = responseString.getBytes();

69
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java Normal file
View File

@ -0,0 +1,69 @@
package org.qortal.arbitrary;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.repository.DataException;
import org.qortal.utils.NTP;
import java.io.IOException;
public class ArbitraryDataBuildQueueItem {
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private Long buildStartTimestamp = null;
private static long BUILD_TIMEOUT = 60*1000L; // 60 seconds
public ArbitraryDataBuildQueueItem(String resourceId, ResourceIdType resourceIdType, Service service) {
this.resourceId = resourceId;
this.resourceIdType = resourceIdType;
this.service = service;
}
public void build() throws IOException, DataException {
Long now = NTP.getTime();
if (now == null) {
throw new IllegalStateException("NTP time hasn't synced yet");
}
this.buildStartTimestamp = now;
ArbitraryDataReader arbitraryDataReader =
new ArbitraryDataReader(this.resourceId, this.resourceIdType, this.service);
// We do not want to overwrite the existing cache, as this will be invalidated
// automatically if new data has arrived
arbitraryDataReader.loadSynchronously(false);
}
public boolean isBuilding() {
return this.buildStartTimestamp != null;
}
public boolean isQueued() {
return this.buildStartTimestamp == null;
}
public boolean hasReachedBuildTimeout(Long now) {
if (now == null || this.buildStartTimestamp == null) {
return true;
}
return now - this.buildStartTimestamp > BUILD_TIMEOUT;
}
public String getResourceId() {
return this.resourceId;
}
public Long getBuildStartTimestamp() {
return this.buildStartTimestamp;
}
@Override
public String toString() {
return String.format("%s %s", this.service, this.resourceId);
}
}

2
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -112,7 +112,7 @@ public class ArbitraryDataBuilder {
String sig58 = Base58.encode(transactionData.getSignature());
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
arbitraryDataReader.setTransactionData(transactionData);
arbitraryDataReader.load(true);
arbitraryDataReader.loadSynchronously(true);
Path path = arbitraryDataReader.getFilePath();
if (path == null) {
throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58));

2
src/main/java/org/qortal/arbitrary/ArbitraryDataCache.java
View File

@ -2,7 +2,7 @@ package org.qortal.arbitrary;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.controller.ArbitraryDataManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException;

50
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.crypto.AES;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@ -59,7 +60,54 @@ public class ArbitraryDataReader {
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
}
public void load(boolean overwrite) throws IllegalStateException, IOException, DataException {
public boolean isCachedDataAvailable() {
// If this resource is in the build queue then we shouldn't attempt to serve
// cached data, as it may not be fully built
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
if (ArbitraryDataManager.getInstance().isInBuildQueue(queueItem)) {
return false;
}
// Not in the build queue - so check the cache itself
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, false,
this.resourceId, this.resourceIdType, this.service);
if (!cache.shouldInvalidate()) {
this.filePath = this.uncompressedPath;
return true;
}
return false;
}
/**
* loadAsynchronously
*
* Attempts to load the resource asynchronously
* This adds the build task to a queue, and the result will be cached when complete
* To check the status of the build, periodically call isCachedDataAvailable()
* Once it returns true, you can then use getFilePath() to access the data itself.
* TODO: create API to check the status
* @return
*/
public boolean loadAsynchronously() {
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(this.resourceId, this.resourceIdType, this.service);
return ArbitraryDataManager.getInstance().addToBuildQueue(queueItem);
}
/**
* loadSynchronously
*
* Attempts to load the resource synchronously
* Warning: this can block for a long time when building or fetching complex data
* If no exception is thrown, you can then use getFilePath() to access the data immediately after returning
*
* @param overwrite - set to true to force rebuild an existing cache
* @throws IllegalStateException
* @throws IOException
* @throws DataException
*/
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException {
try {
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
this.resourceId, this.resourceIdType, this.service);

2
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataCache.java
View File

@ -45,7 +45,7 @@ public class ArbitraryDataMetadataCache extends ArbitraryDataMetadata {
patch.put("timestamp", this.timestamp);
this.jsonString = patch.toString(2);
LOGGER.info("Cache metadata: {}", this.jsonString);
LOGGER.trace("Cache metadata: {}", this.jsonString);
}

3
src/main/java/org/qortal/controller/Controller.java
View File

@ -14,6 +14,7 @@ import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.block.BlockChain.BlockTimingByHeight;
import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.tradebot.TradeBot;
import org.qortal.data.account.MintingAccountData;
import org.qortal.data.account.RewardShareData;
@ -332,7 +333,7 @@ public class Controller extends Thread {
return this.savedArgs;
}
/* package */ static boolean isStopping() {
/* package */ public static boolean isStopping() {
return isStopping;
}

78
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuildManager.java Normal file
View File

@ -0,0 +1,78 @@
package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
import org.qortal.controller.Controller;
import org.qortal.repository.DataException;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.util.Map;
public class ArbitraryDataBuildManager implements Runnable {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataBuildManager.class);
public ArbitraryDataBuildManager() {
}
public void run() {
Thread.currentThread().setName("Arbitrary Data Build Manager");
ArbitraryDataManager arbitraryDataManager = ArbitraryDataManager.getInstance();
while (!Controller.isStopping()) {
try {
Thread.sleep(1000);
if (arbitraryDataManager.arbitraryDataBuildQueue == null) {
continue;
}
if (arbitraryDataManager.arbitraryDataBuildQueue.isEmpty()) {
continue;
}
// Find resources that are queued for building
Map.Entry<String, ArbitraryDataBuildQueueItem> next = arbitraryDataManager.arbitraryDataBuildQueue
.entrySet().stream().filter(e -> e.getValue().isQueued()).findFirst().get();
if (next == null) {
continue;
}
Long now = NTP.getTime();
if (now == null) {
continue;
}
String resourceId = next.getKey();
ArbitraryDataBuildQueueItem queueItem = next.getValue();
if (queueItem == null || queueItem.hasReachedBuildTimeout(now)) {
this.removeFromQueue(resourceId);
}
try {
// Perform the build
LOGGER.info("Building {}...", queueItem);
queueItem.build();
this.removeFromQueue(resourceId);
LOGGER.info("Finished building {}", queueItem);
} catch (IOException | DataException e) {
// Something went wrong - so remove it from the queue
// TODO: we may want to keep track of this in a "cooloff" list to prevent frequent re-attempts
this.removeFromQueue(resourceId);
}
} catch (InterruptedException e) {
// Time to exit
}
}
}
private void removeFromQueue(String resourceId) {
ArbitraryDataManager.getInstance().arbitraryDataBuildQueue.remove(resourceId);
}
}

67
src/main/java/org/qortal/controller/ArbitraryDataManager.java → src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -1,10 +1,14 @@
package org.qortal.controller;
package org.qortal.controller.arbitrary;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
import org.qortal.controller.Controller;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.network.Network;
@ -70,6 +74,11 @@ public class ArbitraryDataManager extends Thread {
*/
private static long ARBITRARY_DATA_CACHE_TIMEOUT = 60 * 60 * 1000L; // 60 minutes
/**
* Map to keep track of arbitrary transaction resources currently being built (or queued).
*/
public Map<String, ArbitraryDataBuildQueueItem> arbitraryDataBuildQueue = Collections.synchronizedMap(new HashMap<>());
private ArbitraryDataManager() {
}
@ -85,6 +94,11 @@ public class ArbitraryDataManager extends Thread {
public void run() {
Thread.currentThread().setName("Arbitrary Data Manager");
// Use a fixed thread pool to execute the arbitrary data build actions (currently just a single thread)
// This can be expanded to have multiple threads processing the build queue when needed
ExecutorService arbitraryDataBuildExecutor = Executors.newFixedThreadPool(1);
arbitraryDataBuildExecutor.execute(new ArbitraryDataBuildManager());
try {
while (!isStopping) {
Thread.sleep(2000);
@ -247,11 +261,62 @@ public class ArbitraryDataManager extends Thread {
this.arbitraryDataCachedResources = new HashMap<>();
}
Long now = NTP.getTime();
if (now == null) {
return;
}
// Set the timestamp to now + the timeout
Long timestamp = NTP.getTime() + ARBITRARY_DATA_CACHE_TIMEOUT;
this.arbitraryDataCachedResources.put(resourceId, timestamp);
}
// Build queue
public boolean addToBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
String resourceId = queueItem.getResourceId();
if (resourceId == null) {
return false;
}
if (this.arbitraryDataBuildQueue == null) {
return false;
}
if (NTP.getTime() == null) {
// Can't use queues until we have synced the time
return false;
}
if (this.arbitraryDataBuildQueue.put(resourceId, queueItem) != null) {
// Already in queue
return true;
}
LOGGER.info("Added {} to build queue", resourceId);
// Added to queue
return true;
}
public boolean isInBuildQueue(ArbitraryDataBuildQueueItem queueItem) {
String resourceId = queueItem.getResourceId();
if (resourceId == null) {
return false;
}
if (this.arbitraryDataBuildQueue == null) {
return false;
}
if (this.arbitraryDataBuildQueue.containsKey(resourceId)) {
// Already in queue
return true;
}
// Not in queue
return false;
}
// Network handlers

23
src/main/resources/loading/index.html Normal file
View File

@ -0,0 +1,23 @@
<html>
<head>
<title>Loading...</title>
<script>
setTimeout(window.location.reload.bind(window.location), 1);
</script>
<style>
body {
font-family: Arial;
text-align: center;
}
h1 {
margin-top: 50px;
}
</style>
</head>
</html>
<body>
<h1>Loading... please wait...</h1>
<p>This page will refresh automatically when the content becomes available</p>
<p>(We can show a Qortal branded loading screen here)</p>
</body>
</html>