From 844501d6cd08bfb83334961b6c0518a8fd6ad739 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Fri, 19 Nov 2021 15:26:52 +0000 Subject: [PATCH] Added GET /arbitrary/resource/status/* API endpoints These can be used to check the current status of a resource. The different statuses are: NOT_STARTED, DOWNLOADING DOWNLOADED BUILDING READY DOWNLOAD_FAILED BUILD_FAILED UNSUPPORTED Not all statuses are returned yet. The build process needs more functionality to be able to support DOWNLOADED and DOWNLOAD_FAILED. Also, BUILDING and BUILD_FAILED are currently unable to distinguish between different resources with the same registered name, so need some attention. --- .../api/model/ArbitraryResourceSummary.java | 29 ++++++++ .../api/resource/ArbitraryResource.java | 41 ++++++++++- .../arbitrary/ArbitraryDataBuilder.java | 41 ++++++++++- .../qortal/arbitrary/ArbitraryDataReader.java | 28 ++++++-- .../arbitrary/ArbitraryDataResource.java | 69 +++++++++++++++++++ 5 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java diff --git a/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java new file mode 100644 index 00000000..2ad69933 --- /dev/null +++ b/src/main/java/org/qortal/api/model/ArbitraryResourceSummary.java @@ -0,0 +1,29 @@ +package org.qortal.api.model; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; + +@XmlAccessorType(XmlAccessType.FIELD) +public class ArbitraryResourceSummary { + + public enum ArbitraryResourceStatus { + NOT_STARTED, + DOWNLOADING, + DOWNLOADED, + BUILDING, + READY, + DOWNLOAD_FAILED, + BUILD_FAILED, + UNSUPPORTED + } + + public ArbitraryResourceStatus status; + + public ArbitraryResourceSummary() { + } + + public ArbitraryResourceSummary(ArbitraryResourceStatus status) { + this.status = status; + } + +} diff --git a/src/main/java/org/qortal/api/resource/ArbitraryResource.java b/src/main/java/org/qortal/api/resource/ArbitraryResource.java index 8a800092..7b76264a 100644 --- a/src/main/java/org/qortal/api/resource/ArbitraryResource.java +++ b/src/main/java/org/qortal/api/resource/ArbitraryResource.java @@ -30,9 +30,10 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.api.*; +import org.qortal.api.model.ArbitraryResourceSummary; import org.qortal.api.resource.TransactionsResource.ConfirmationStatus; -import org.qortal.arbitrary.ArbitraryDataReader; -import org.qortal.arbitrary.ArbitraryDataTransactionBuilder; +import org.qortal.arbitrary.*; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; import org.qortal.arbitrary.exception.MissingDataException; import org.qortal.arbitrary.misc.Service; import org.qortal.controller.Controller; @@ -46,7 +47,6 @@ import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; import org.qortal.settings.Settings; -import org.qortal.arbitrary.ArbitraryDataFile; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.ValidationResult; @@ -98,6 +98,41 @@ public class ArbitraryResource { } } + @GET + @Path("/resource/status/{service}/{name}") + @Operation( + summary = "Get status of arbitrary resource with supplied service and name", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class)) + ) + } + ) + public ArbitraryResourceSummary getDefaultResourceStatus(@PathParam("service") Service service, + @PathParam("name") String name) { + + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, null); + return resource.getSummary(); + } + + @GET + @Path("/resource/status/{service}/{name}/{identifier}") + @Operation( + summary = "Get status of arbitrary resource with supplied service, name and identifier", + responses = { + @ApiResponse( + content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ArbitraryResourceSummary.class)) + ) + } + ) + public ArbitraryResourceSummary getResourceStatus(@PathParam("service") Service service, + @PathParam("name") String name, + @PathParam("identifier") String identifier) { + + ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier); + return resource.getSummary(); + } + @GET @Path("/search") diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index 241afdae..7a4c4de6 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -30,6 +30,8 @@ public class ArbitraryDataBuilder { private Service service; private String identifier; + private boolean canRequestMissingFiles; + private List transactions; private ArbitraryTransactionData latestPutTransaction; private List paths; @@ -42,14 +44,37 @@ public class ArbitraryDataBuilder { this.service = service; this.identifier = identifier; this.paths = new ArrayList<>(); + + // By default we can request missing files + // Callers can use setCanRequestMissingFiles(false) to prevent it + this.canRequestMissingFiles = true; } - public void build() throws DataException, IOException, MissingDataException { + /** + * Process transactions, but do not build anything + * This is useful for checking the status of a given resource + * + * @throws DataException + * @throws IOException + * @throws MissingDataException + */ + public void process() throws DataException, IOException, MissingDataException { this.fetchTransactions(); this.validateTransactions(); this.processTransactions(); this.validatePaths(); this.findLatestSignature(); + } + + /** + * Build the latest state of a given resource + * + * @throws DataException + * @throws IOException + * @throws MissingDataException + */ + public void build() throws DataException, IOException, MissingDataException { + this.process(); this.buildLatestState(); this.cacheLatestSignature(); } @@ -124,6 +149,7 @@ public class ArbitraryDataBuilder { ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service, this.identifier); arbitraryDataReader.setTransactionData(transactionData); + arbitraryDataReader.setCanRequestMissingFiles(this.canRequestMissingFiles); boolean hasMissingData = false; try { arbitraryDataReader.loadSynchronously(true); @@ -134,6 +160,9 @@ public class ArbitraryDataBuilder { // Handle missing data if (hasMissingData) { + if (!this.canRequestMissingFiles) { + throw new MissingDataException("Files are missing but were not requested."); + } if (count == transactionDataList.size()) { // This is the final transaction in the list, so we need to fail throw new MissingDataException("Requesting missing files. Please wait and try again."); @@ -235,4 +264,14 @@ public class ArbitraryDataBuilder { return this.layerCount; } + /** + * Use the below setter to ensure that we only read existing + * data without requesting any missing files, + * + * @param canRequestMissingFiles + */ + public void setCanRequestMissingFiles(boolean canRequestMissingFiles) { + this.canRequestMissingFiles = canRequestMissingFiles; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java index 9d450a76..fa29c849 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java @@ -27,7 +27,6 @@ import javax.crypto.IllegalBlockSizeException; import javax.crypto.NoSuchPaddingException; import javax.crypto.SecretKey; import javax.crypto.spec.SecretKeySpec; -import javax.xml.crypto.Data; import java.io.File; import java.io.IOException; import java.io.InvalidObjectException; @@ -49,6 +48,7 @@ public class ArbitraryDataReader { private ArbitraryTransactionData transactionData; private String secret58; private Path filePath; + private boolean canRequestMissingFiles; // Intermediate paths private Path workingPath; @@ -77,6 +77,10 @@ public class ArbitraryDataReader { this.workingPath = this.buildWorkingPath(); this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data"); + + // By default we can request missing files + // Callers can use setCanRequestMissingFiles(false) to prevent it + this.canRequestMissingFiles = true; } private Path buildWorkingPath() { @@ -318,14 +322,18 @@ public class ArbitraryDataReader { } else { // Ask the arbitrary data manager to fetch data for this transaction - boolean requested = ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature()); String message; + if (this.canRequestMissingFiles) { + boolean requested = ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature()); - if (requested) { - message = String.format("Requested missing data for file %s", arbitraryDataFile); + if (requested) { + message = String.format("Requested missing data for file %s", arbitraryDataFile); + } else { + message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile); + } } else { - message = String.format("Unable to reissue request for missing file %s due to rate limit. Please try again later.", arbitraryDataFile); + message = String.format("Missing data for file %s", arbitraryDataFile); } // Throw a missing data exception, which allows subsequent layers to fetch data @@ -503,4 +511,14 @@ public class ArbitraryDataReader { return this.latestSignature; } + /** + * Use the below setter to ensure that we only read existing + * data without requesting any missing files, + * + * @param canRequestMissingFiles + */ + public void setCanRequestMissingFiles(boolean canRequestMissingFiles) { + this.canRequestMissingFiles = canRequestMissingFiles; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java new file mode 100644 index 00000000..3bcaec92 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java @@ -0,0 +1,69 @@ +package org.qortal.arbitrary; + +import org.qortal.api.model.ArbitraryResourceSummary; +import org.qortal.api.model.ArbitraryResourceSummary.ArbitraryResourceStatus; +import org.qortal.arbitrary.ArbitraryDataFile.ResourceIdType; +import org.qortal.arbitrary.exception.MissingDataException; +import org.qortal.arbitrary.misc.Service; +import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; +import org.qortal.repository.DataException; + +import java.io.IOException; + +public class ArbitraryDataResource { + + private String resourceId; + private ResourceIdType resourceIdType; + private Service service; + private String identifier; + + public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) { + this.resourceId = resourceId; + this.resourceIdType = resourceIdType; + this.service = service; + this.identifier = identifier; + } + + public ArbitraryResourceSummary getSummary() { + if (resourceIdType != ResourceIdType.NAME) { + // We only support statuses for resources with a name + return new ArbitraryResourceSummary(ArbitraryResourceStatus.UNSUPPORTED); + } + + // Firstly check the cache to see if it's already built + ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader( + resourceId, resourceIdType, service, identifier); + if (arbitraryDataReader.isCachedDataAvailable()) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.READY); + } + + // Next check if there's a build in progress + ArbitraryDataBuildQueueItem queueItem = + new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier); + if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) { // TODO: currently keyed by name only + return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILDING); + } + + // Check if a build has failed + if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) { // TODO: currently keyed by name only + return new ArbitraryResourceSummary(ArbitraryResourceStatus.BUILD_FAILED); + } + + // Check if we have all data locally for this resource + ArbitraryDataBuilder builder = new ArbitraryDataBuilder(resourceId, service, identifier); + builder.setCanRequestMissingFiles(false); + try { + builder.process(); + + } catch (MissingDataException e) { + return new ArbitraryResourceSummary(ArbitraryResourceStatus.DOWNLOADING); + + } catch (IOException | DataException e) { + // Ignore for now + } + + // FUTURE: support DOWNLOADED state once the build queue system has been upgraded + + return new ArbitraryResourceSummary(ArbitraryResourceStatus.NOT_STARTED); + } +}