AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Added MissingDataException

Browse Source 
This is generated whenever a data resource cannot be built because it is missing data for at least one layer. Using a custom exception type here enables a few new features:

1. A single build process is now able to request missing data from all the layers that need it. Previously it would only request from the first missing layer and would then give up. This resulted in the user/application having to issue the build command multiple times rather than just once, until all layers had been requested.

2. GET /arbitrary/{service}/{name} will now block the response and retry in the background until the data arrives. This allows it to be used synchronously. Note: we'll need to add a timeout.

3. Loading a website via GET /site/{name} will avoid adding to the failed builds queue when a MissingDataException is thrown, which allows it to be quickly retried. The interface already auto refreshes, allowing the site to load as soon as it's available.
This commit is contained in:
CalDescent 2021-11-04 09:09:54 +00:00
parent ce15784851
commit 09a7fcaba4
10 changed files with 87 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -30,6 +30,8 @@ import org.qortal.api.*;
import org.qortal.api.resource.TransactionsResource.ConfirmationStatus;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.controller.Controller;
import org.qortal.data.account.AccountData;
import org.qortal.data.naming.NameData;
import org.qortal.data.transaction.ArbitraryTransactionData;
@ -259,7 +261,16 @@ public class ArbitraryResource {
Service service = Service.valueOf(serviceString);
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(name, ArbitraryDataFile.ResourceIdType.NAME, service);
try {
arbitraryDataReader.loadSynchronously(rebuild);
// Loop until we have data
while (!Controller.isStopping()) {
try {
arbitraryDataReader.loadSynchronously(rebuild);
break;
} catch (MissingDataException e) {
continue;
}
}
// TODO: limit file size that can be read into memory
java.nio.file.Path path = Paths.get(arbitraryDataReader.getFilePath().toString(), filepath);

3
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@ -31,6 +31,7 @@ import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.HTMLParser;
import org.qortal.api.Security;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException;
@ -145,7 +146,7 @@ public class WebsiteResource {
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(Paths.get(directoryPath), name, service, method, compression);
try {
arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException e) {
} catch (IOException | DataException | InterruptedException | MissingDataException e) {
LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
} catch (RuntimeException e) {

3
src/main/java/org/qortal/arbitrary/ArbitraryDataBuildQueueItem.java
View File

@ -1,5 +1,6 @@
package org.qortal.arbitrary;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.repository.DataException;
@ -30,7 +31,7 @@ public class ArbitraryDataBuildQueueItem {
this.creationTimestamp = NTP.getTime();
}
public void build() throws IOException, DataException {
public void build() throws IOException, DataException, MissingDataException {
Long now = NTP.getTime();
if (now == null) {
throw new IllegalStateException("NTP time hasn't synced yet");

27
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -2,6 +2,7 @@ package org.qortal.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataCache;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Method;
@ -40,7 +41,7 @@ public class ArbitraryDataBuilder {
this.paths = new ArrayList<>();
}
public void build() throws DataException, IOException {
public void build() throws DataException, IOException, MissingDataException {
this.fetchTransactions();
this.validateTransactions();
this.processTransactions();
@ -104,17 +105,37 @@ public class ArbitraryDataBuilder {
}
}
private void processTransactions() throws IOException, DataException {
private void processTransactions() throws IOException, DataException, MissingDataException {
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
int count = 0;
for (ArbitraryTransactionData transactionData : transactionDataList) {
LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature()));
count++;
// Build the data file, overwriting anything that was previously there
String sig58 = Base58.encode(transactionData.getSignature());
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
arbitraryDataReader.setTransactionData(transactionData);
arbitraryDataReader.loadSynchronously(true);
boolean hasMissingData = false;
try {
arbitraryDataReader.loadSynchronously(true);
}
catch (MissingDataException e) {
hasMissingData = true;
}
// Handle missing data
if (hasMissingData) {
if (count == transactionDataList.size()) {
// This is the final transaction in the list, so we need to fail
throw new MissingDataException("Requesting missing files. Please wait and try again.");
}
// There are more transactions, so we should process them to give them the opportunity to request data
continue;
}
// By this point we should have all data needed to build the layers
Path path = arbitraryDataReader.getFilePath();
if (path == null) {
throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58));

20
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.crypto.AES;
@ -33,6 +34,7 @@ import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.MissingResourceException;
public class ArbitraryDataReader {
@ -113,7 +115,7 @@ public class ArbitraryDataReader {
* @throws IOException
* @throws DataException
*/
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException {
public void loadSynchronously(boolean overwrite) throws IllegalStateException, IOException, DataException, MissingDataException {
try {
ArbitraryDataCache cache = new ArbitraryDataCache(this.uncompressedPath, overwrite,
this.resourceId, this.resourceIdType, this.service);
@ -197,7 +199,7 @@ public class ArbitraryDataReader {
}
}
private void fetch() throws IllegalStateException, IOException, DataException {
private void fetch() throws IllegalStateException, IOException, DataException, MissingDataException {
switch (resourceIdType) {
case FILE_HASH:
@ -228,7 +230,7 @@ public class ArbitraryDataReader {
this.filePath = arbitraryDataFile.getFilePath();
}
private void fetchFromName() throws IllegalStateException, IOException, DataException {
private void fetchFromName() throws IllegalStateException, IOException, DataException, MissingDataException {
try {
// Build the existing state using past transactions
@ -250,7 +252,7 @@ public class ArbitraryDataReader {
}
}
private void fetchFromSignature() throws IllegalStateException, IOException, DataException {
private void fetchFromSignature() throws IllegalStateException, IOException, DataException, MissingDataException {
// Load the full transaction data from the database so we can access the file hashes
ArbitraryTransactionData transactionData;
@ -264,7 +266,7 @@ public class ArbitraryDataReader {
this.fetchFromTransactionData(transactionData);
}
private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, DataException {
private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, MissingDataException {
if (!(transactionData instanceof ArbitraryTransactionData)) {
throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId));
}
@ -287,10 +289,10 @@ public class ArbitraryDataReader {
// Ask the arbitrary data manager to fetch data for this transaction
ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature());
// Fail the build, as it will be retried later once the chunks arrive
String response = String.format("Missing chunks for file %s have been requested. Please try again once they have been received.", arbitraryDataFile);
LOGGER.info(response);
throw new IllegalStateException(response);
// Throw a missing data exception, which allows subsequent layers to fetch data
String message = String.format("Requested missing data for file %s", arbitraryDataFile);
LOGGER.info(message);
throw new MissingDataException(message);
}
// We have all the chunks but not the complete file, so join them
arbitraryDataFile.addChunkHashes(chunkHashes);

3
src/main/java/org/qortal/arbitrary/ArbitraryDataTransactionBuilder.java
View File

@ -2,6 +2,7 @@ package org.qortal.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
@ -68,7 +69,7 @@ public class ArbitraryDataTransactionBuilder {
ArbitraryDataWriter arbitraryDataWriter = new ArbitraryDataWriter(path, name, service, method, compression);
try {
arbitraryDataWriter.save();
} catch (IOException | DataException | InterruptedException | RuntimeException e) {
} catch (IOException | DataException | InterruptedException | RuntimeException | MissingDataException e) {
LOGGER.info("Unable to create arbitrary data file: {}", e.getMessage());
throw new DataException(String.format("Unable to create arbitrary data file: %s", e.getMessage()));
}

7
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -3,6 +3,7 @@ package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.crypto.Crypto;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.crypto.AES;
@ -52,7 +53,7 @@ public class ArbitraryDataWriter {
this.compression = compression;
}
public void save() throws IllegalStateException, IOException, DataException, InterruptedException {
public void save() throws IllegalStateException, IOException, DataException, InterruptedException, MissingDataException {
try {
this.preExecute();
this.process();
@ -94,7 +95,7 @@ public class ArbitraryDataWriter {
this.workingPath = tempDir;
}
private void process() throws DataException, IOException {
private void process() throws DataException, IOException, MissingDataException {
switch (this.method) {
case PUT:
@ -110,7 +111,7 @@ public class ArbitraryDataWriter {
}
}
private void processPatch() throws DataException, IOException {
private void processPatch() throws DataException, IOException, MissingDataException {
// Build the existing state using past transactions
ArbitraryDataBuilder builder = new ArbitraryDataBuilder(this.name, this.service);

20
src/main/java/org/qortal/arbitrary/exception/MissingDataException.java Normal file
View File

@ -0,0 +1,20 @@
package org.qortal.arbitrary.exception;
public class MissingDataException extends Exception {
public MissingDataException() {
}
public MissingDataException(String message) {
super(message);
}
public MissingDataException(String message, Throwable cause) {
super(message, cause);
}
public MissingDataException(Throwable cause) {
super(cause);
}
}

7
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataBuilderThread.java
View File

@ -3,6 +3,7 @@ package org.qortal.controller.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataBuildQueueItem;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.controller.Controller;
import org.qortal.repository.DataException;
import org.qortal.utils.NTP;
@ -69,6 +70,12 @@ public class ArbitraryDataBuilderThread implements Runnable {
this.removeFromQueue(resourceId);
LOGGER.info("Finished building {}", queueItem);
} catch (MissingDataException e) {
LOGGER.info("Missing data for {}: {}", queueItem, e.getMessage());
queueItem.setFailed(true);
this.removeFromQueue(resourceId);
// Don't add to the failed builds list, as we may want to retry sooner
} catch (IOException | DataException | RuntimeException e) {
LOGGER.info("Error building {}: {}", queueItem, e.getMessage());
// Something went wrong - so remove it from the queue, and add to failed builds list

5
src/test/java/org/qortal/test/arbitrary/ArbitraryDataTests.java
View File

@ -7,6 +7,7 @@ import org.qortal.arbitrary.ArbitraryDataDigest;
import org.qortal.arbitrary.ArbitraryDataFile.*;
import org.qortal.arbitrary.ArbitraryDataReader;
import org.qortal.arbitrary.ArbitraryDataTransactionBuilder;
import org.qortal.arbitrary.exception.MissingDataException;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
@ -37,7 +38,7 @@ public class ArbitraryDataTests extends Common {
}
@Test
public void testCombineMultipleLayers() throws DataException, IOException {
public void testCombineMultipleLayers() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());
@ -159,7 +160,7 @@ public class ArbitraryDataTests extends Common {
}
@Test
public void testUpdateResource() throws DataException, IOException {
public void testUpdateResource() throws DataException, IOException, MissingDataException {
try (final Repository repository = RepositoryManager.getRepository()) {
PrivateKeyAccount alice = Common.getTestAccount(repository, "alice");
String publicKey58 = Base58.encode(alice.getPublicKey());