crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Added GET /arbitrary/resource/status/* API endpoints

Browse Source 
These can be used to check the current status of a resource. The different statuses are:

NOT_STARTED,
DOWNLOADING
DOWNLOADED
BUILDING
READY
DOWNLOAD_FAILED
BUILD_FAILED
UNSUPPORTED

Not all statuses are returned yet. The build process needs more functionality to be able to support DOWNLOADED and DOWNLOAD_FAILED. Also, BUILDING and BUILD_FAILED are currently unable to distinguish between different resources with the same registered name, so need some attention.
This commit is contained in:
CalDescent 2021-11-19 15:26:52 +00:00
parent 020bd00b8f
commit 844501d6cd
5 changed files with 199 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java Normal file
View File

@ -0,0 +1,29 @@
package org.qortal.api.model;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@XmlAccessorType(XmlAccessType.FIELD)
public class ArbitraryResourceSummary {
public enum ArbitraryResourceStatus {
NOT_STARTED,
DOWNLOADING,
DOWNLOADED,
BUILDING,
READY,
DOWNLOAD_FAILED,
BUILD_FAILED,
UNSUPPORTED
}
public ArbitraryResourceStatus status;
public ArbitraryResourceSummary() {
}
public ArbitraryResourceSummary(ArbitraryResourceStatus status) {
this.status = status;
}
}

41
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -30,9 +30,10 @@ import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.api.*;
import org.qortal.api.model.ArbitraryResourceSummary;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.*;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.Controller;
@ -46,7 +47,6 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings;
import org.qortal.arbitrary.ArbitraryDataFile;
import org.qortal.transaction.Transaction;
import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.transaction.Transaction.ValidationResult;
@ -98,6 +98,41 @@ public class ArbitraryResource {
}
}
@GET
@Path("/resource/status/{service}/{name}")
@Operation(
summary = "Get status of arbitrary resource with supplied service and name",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class))
)
}
)
public ArbitraryResourceSummary getDefaultResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, null);
return resource.getSummary();
}
@GET
@Path("/resource/status/{service}/{name}/{identifier}")
@Operation(
summary = "Get status of arbitrary resource with supplied service, name and identifier",
responses = {
@ApiResponse(
content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class))
)
}
)
public ArbitraryResourceSummary getResourceStatus(@PathParam("service") Service service,
@PathParam("name") String name,
@PathParam("identifier") String identifier) {
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getSummary();
}
@GET
@Path("/search")

41
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -30,6 +30,8 @@ public class ArbitraryDataBuilder {
private Service service;
private String identifier;
private boolean canRequestMissingFiles;
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
private List<Path> paths;
@ -42,14 +44,37 @@ public class ArbitraryDataBuilder {
this.service = service;
this.identifier = identifier;
this.paths = new ArrayList<>();
// By default we can request missing files
// Callers can use setCanRequestMissingFiles(false) to prevent it
this.canRequestMissingFiles = true;
}
public void build() throws DataException, IOException, MissingDataException {
/**
* Process transactions, but do not build anything
* This is useful for checking the status of a given resource
*
* @throws DataException
* @throws IOException
* @throws MissingDataException
*/
public void process() throws DataException, IOException, MissingDataException {
this.fetchTransactions();
this.validateTransactions();
this.processTransactions();
this.validatePaths();
this.findLatestSignature();
}
/**
* Build the latest state of a given resource
*
* @throws DataException
* @throws IOException
* @throws MissingDataException
*/
public void build() throws DataException, IOException, MissingDataException {
this.process();
this.buildLatestState();
this.cacheLatestSignature();
}
@ -124,6 +149,7 @@ public class ArbitraryDataBuilder {
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA,
this.service, this.identifier);
arbitraryDataReader.setTransactionData(transactionData);
arbitraryDataReader.setCanRequestMissingFiles(this.canRequestMissingFiles);
boolean hasMissingData = false;
try {
arbitraryDataReader.loadSynchronously(true);
@ -134,6 +160,9 @@ public class ArbitraryDataBuilder {
// Handle missing data
if (hasMissingData) {
if (!this.canRequestMissingFiles) {
throw new MissingDataException("Files are missing but were not requested.");
}
if (count == transactionDataList.size()) {
// This is the final transaction in the list, so we need to fail
throw new MissingDataException("Requesting missing files. Please wait and try again.");
@ -235,4 +264,14 @@ public class ArbitraryDataBuilder {
return this.layerCount;
}
/**
* Use the below setter to ensure that we only read existing
* data without requesting any missing files,
*
* @param canRequestMissingFiles
*/
public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
this.canRequestMissingFiles = canRequestMissingFiles;
}
}

28
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -27,7 +27,6 @@ import javax.crypto.IllegalBlockSizeException;
import javax.crypto.NoSuchPaddingException;
import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import javax.xml.crypto.Data;
import java.io.File;
import java.io.IOException;
import java.io.InvalidObjectException;
@ -49,6 +48,7 @@ public class ArbitraryDataReader {
private ArbitraryTransactionData transactionData;
private String secret58;
private Path filePath;
private boolean canRequestMissingFiles;
// Intermediate paths
private Path workingPath;
@ -77,6 +77,10 @@ public class ArbitraryDataReader {
this.workingPath = this.buildWorkingPath();
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
// By default we can request missing files
// Callers can use setCanRequestMissingFiles(false) to prevent it
this.canRequestMissingFiles = true;
}
private Path buildWorkingPath() {
@ -318,14 +322,18 @@ public class ArbitraryDataReader {
}
else {
// Ask the arbitrary data manager to fetch data for this transaction
boolean requested = ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature());
String message;
if (this.canRequestMissingFiles) {
boolean requested = ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature());
if (requested) {
message = String.format("Requested missing data for file %s", arbitraryDataFile);
if (requested) {
message = String.format("Requested missing data for file %s", arbitraryDataFile);
} else {
message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile);
}
}
else {
message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile);
message = String.format("Missing data for file %s", arbitraryDataFile);
}
// Throw a missing data exception, which allows subsequent layers to fetch data
@ -503,4 +511,14 @@ public class ArbitraryDataReader {
return this.latestSignature;
}
/**
* Use the below setter to ensure that we only read existing
* data without requesting any missing files,
*
* @param canRequestMissingFiles
*/
public void setCanRequestMissingFiles(boolean canRequestMissingFiles) {
this.canRequestMissingFiles = canRequestMissingFiles;
}
}

69
src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java Normal file
View File

@ -0,0 +1,69 @@
package org.qortal.arbitrary;
import org.qortal.api.model.ArbitraryResourceSummary;
import org.qortal.api.model.ArbitraryResourceSummary.ArbitraryResourceStatus;
import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.misc.Service;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.repository.DataException;
import java.io.IOException;
public class ArbitraryDataResource {
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private String identifier;
public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
this.resourceId = resourceId;
this.resourceIdType = resourceIdType;
this.service = service;
this.identifier = identifier;
}
public ArbitraryResourceSummary getSummary() {
if (resourceIdType != ResourceIdType.NAME) {
// We only support statuses for resources with a name
return new ArbitraryResourceSummary(ArbitraryResourceStatus.UNSUPPORTED);
}
// Firstly check the cache to see if it's already built
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
resourceId, resourceIdType, service, identifier);
if (arbitraryDataReader.isCachedDataAvailable()) {
return new ArbitraryResourceSummary(ArbitraryResourceStatus.READY);
}
// Next check if there's a build in progress
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { // TODO: currently keyed by name only
return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILDING);
}
// Check if a build has failed
if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { // TODO: currently keyed by name only
return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILD_FAILED);
}
// Check if we have all data locally for this resource
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(resourceId, service, identifier);
builder.setCanRequestMissingFiles(false);
try {
builder.process();
} catch (MissingDataException e) {
return new ArbitraryResourceSummary(ArbitraryResourceStatus.DOWNLOADING);
} catch (IOException | DataException e) {
// Ignore for now
}
// FUTURE: support DOWNLOADED state once the build queue system has been upgraded
return new ArbitraryResourceSummary(ArbitraryResourceStatus.NOT_STARTED);
}
}