AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

When accessing a website or other data resource, request the chunks if we don't already have them.

Browse Source 
This causes the build to fail on the first pass due to missing chunks, however it now fails with a message indicating that it should be retried shortly. The website loader is already set up in such a way that it will be automatically retried, during which time the loading screen is shown.

Also added code to remove the resource from the "failed builds list" once the chunks arrive, so that it is able to be rebuilt sooner than the FAILURE_TIMEOUT (currently 5 minutes).
This commit is contained in:
CalDescent 2021-11-02 19:32:48 +00:00
parent 1002acb021
commit 16bcba6e2e
2 changed files with 27 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/org/qortal/arbitrary/ArbitraryDataReader.java
View File

@ -282,9 +282,14 @@ public class ArbitraryDataReader {
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
if (!arbitraryDataFile.exists()) { if (!arbitraryDataFile.exists()) {
if (!arbitraryDataFile.allChunksExist(chunkHashes)) { if (!arbitraryDataFile.allChunksExist(chunkHashes)) {
// TODO: fetch them?
LOGGER.info(String.format("Missing chunks for file %s", arbitraryDataFile)); // Ask the arbitrary data manager to fetch data for this transaction
throw new IllegalStateException(String.format("Missing chunks for file %s", arbitraryDataFile)); ArbitraryDataManager.getInstance().fetchDataForSignature(transactionData.getSignature());
// Fail the build, as it will be retried later once the chunks arrive
String response = String.format("Missing chunks for file %s have been requested. Please try again once they have been received.", arbitraryDataFile);
LOGGER.info(response);
throw new IllegalStateException(response);
} }
// We have all the chunks but not the complete file, so join them // We have all the chunks but not the complete file, so join them
arbitraryDataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);

20
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -199,7 +199,7 @@ public class ArbitraryDataManager extends Thread {
// Ask our connected peers if they have files for this signature // Ask our connected peers if they have files for this signature
// This process automatically then fetches the files themselves if a peer is found // This process automatically then fetches the files themselves if a peer is found
fetchArbitraryDataFileList(signature); fetchDataForSignature(signature);
} catch (DataException e) { } catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e); LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
@ -238,7 +238,11 @@ public class ArbitraryDataManager extends Thread {
} }
} }
private boolean fetchArbitraryDataFileList(byte[] signature) throws InterruptedException { public boolean fetchDataForSignature(byte[] signature) {
return this.fetchArbitraryDataFileList(signature);
}
private boolean fetchArbitraryDataFileList(byte[] signature) {
LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature))); LOGGER.info(String.format("Sending data file list request for signature %s", Base58.encode(signature)));
// Build request // Build request
Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature); Message getArbitraryDataFileListMessage = new GetArbitraryDataFileListMessage(signature);
@ -264,7 +268,11 @@ public class ArbitraryDataManager extends Thread {
final long singleWait = 100; final long singleWait = 100;
long totalWait = 0; long totalWait = 0;
while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { while (totalWait < ARBITRARY_REQUEST_TIMEOUT) {
try {
Thread.sleep(singleWait); Thread.sleep(singleWait);
} catch (InterruptedException e) {
break;
}
requestEntry = arbitraryDataFileListRequests.get(id); requestEntry = arbitraryDataFileListRequests.get(id);
if (requestEntry == null) if (requestEntry == null)
@ -595,9 +603,17 @@ public class ArbitraryDataManager extends Thread {
// data cache so that it is rebuilt the next time we serve it // data cache so that it is rebuilt the next time we serve it
if (arbitraryTransactionData.getName() != null) { if (arbitraryTransactionData.getName() != null) {
String resourceId = arbitraryTransactionData.getName().toLowerCase(); String resourceId = arbitraryTransactionData.getName().toLowerCase();
LOGGER.info("We have all data for transaction {}", Base58.encode(transactionData.getSignature()));
LOGGER.info("Clearing cache for name {}...", arbitraryTransactionData.getName());
if (this.arbitraryDataCachedResources.containsKey(resourceId)) { if (this.arbitraryDataCachedResources.containsKey(resourceId)) {
this.arbitraryDataCachedResources.remove(resourceId); this.arbitraryDataCachedResources.remove(resourceId);
} }
// Also remove from the failed builds queue in case it previously failed due to missing chunks
if (this.arbitraryDataFailedBuilds.containsKey(resourceId)) {
this.arbitraryDataFailedBuilds.remove(resourceId);
}
} }
// We also need to broadcast to the network that we are now hosting files for this transaction // We also need to broadcast to the network that we are now hosting files for this transaction