AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Include "localChunkCount" and "totalChunkCount" in the GET /arbitrary/resource/status/* API responses.

Browse Source 
These values are left out of other API endpoints where multiple resources are returned, because calculating the chunk counts is too time consuming.
This commit is contained in:
CalDescent 2022-02-01 22:02:14 +00:00
parent 45f2d7ab70
commit 82fa6a4fd8
5 changed files with 102 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/main/java/org/qortal/api/gateway/resource/GatewayResource.java
View File

@ -65,7 +65,7 @@ public class GatewayResource {
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus();
return resource.getStatus(false);
}

4
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -1097,7 +1097,7 @@ public class ArbitraryResource {
}
ArbitraryDataResource resource = new ArbitraryDataResource(name, ResourceIdType.NAME, service, identifier);
return resource.getStatus();
return resource.getStatus(false);
}
private List<ArbitraryResourceInfo> addStatusToResources(List<ArbitraryResourceInfo> resources) {
@ -1106,7 +1106,7 @@ public class ArbitraryResource {
for (ArbitraryResourceInfo resourceInfo : resources) {
ArbitraryDataResource resource = new ArbitraryDataResource(resourceInfo.name, ResourceIdType.NAME,
resourceInfo.service, resourceInfo.identifier);
ArbitraryResourceStatus status = resource.getStatus();
ArbitraryResourceStatus status = resource.getStatus(true);
if (status != null) {
resourceInfo.status = status;
}

73
src/main/java/org/qortal/arbitrary/ArbitraryDataResource.java
View File

@ -38,6 +38,8 @@ public class ArbitraryDataResource {
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
private int layerCount;
private Integer localChunkCount = null;
private Integer totalChunkCount = null;
public ArbitraryDataResource(String resourceId, ResourceIdType resourceIdType, Service service, String identifier) {
this.resourceId = resourceId.toLowerCase();
@ -51,50 +53,56 @@ public class ArbitraryDataResource {
this.identifier = identifier;
}
public ArbitraryResourceStatus getStatus() {
public ArbitraryResourceStatus getStatus(boolean quick) {
// Calculate the chunk counts
// Avoid this for "quick" statuses, to speed things up
if (!quick) {
this.calculateChunkCounts();
}
if (resourceIdType != ResourceIdType.NAME) {
// We only support statuses for resources with a name
return new ArbitraryResourceStatus(Status.UNSUPPORTED);
return new ArbitraryResourceStatus(Status.UNSUPPORTED, this.localChunkCount, this.totalChunkCount);
}
// Check if the name is blocked
if (ResourceListManager.getInstance()
.listContains("blockedNames", this.resourceId, false)) {
return new ArbitraryResourceStatus(Status.BLOCKED);
return new ArbitraryResourceStatus(Status.BLOCKED, this.localChunkCount, this.totalChunkCount);
}
// Check if a build has failed
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILD_FAILED, this.localChunkCount, this.totalChunkCount);
}
// Firstly check the cache to see if it's already built
ArbitraryDataReader arbitraryDataReader = new ArbitraryDataReader(
resourceId, resourceIdType, service, identifier);
if (arbitraryDataReader.isCachedDataAvailable()) {
return new ArbitraryResourceStatus(Status.READY);
}
// Next check if there's a build in progress
ArbitraryDataBuildQueueItem queueItem =
new ArbitraryDataBuildQueueItem(resourceId, resourceIdType, service, identifier);
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILDING);
}
// Check if a build has failed
if (ArbitraryDataBuildManager.getInstance().isInFailedBuildsList(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILD_FAILED);
return new ArbitraryResourceStatus(Status.READY, this.localChunkCount, this.totalChunkCount);
}
// Check if we have all data locally for this resource
if (!this.allFilesDownloaded()) {
if (this.isDownloading()) {
return new ArbitraryResourceStatus(Status.DOWNLOADING);
return new ArbitraryResourceStatus(Status.DOWNLOADING, this.localChunkCount, this.totalChunkCount);
}
else if (this.isDataPotentiallyAvailable()) {
return new ArbitraryResourceStatus(Status.PUBLISHED);
return new ArbitraryResourceStatus(Status.PUBLISHED, this.localChunkCount, this.totalChunkCount);
}
return new ArbitraryResourceStatus(Status.MISSING_DATA);
return new ArbitraryResourceStatus(Status.MISSING_DATA, this.localChunkCount, this.totalChunkCount);
}
// Check if there's a build in progress
if (ArbitraryDataBuildManager.getInstance().isInBuildQueue(queueItem)) {
return new ArbitraryResourceStatus(Status.BUILDING, this.localChunkCount, this.totalChunkCount);
}
// We have all data locally
return new ArbitraryResourceStatus(Status.DOWNLOADED);
return new ArbitraryResourceStatus(Status.DOWNLOADED, this.localChunkCount, this.totalChunkCount);
}
public boolean delete() {
@ -147,6 +155,12 @@ public class ArbitraryDataResource {
}
private boolean allFilesDownloaded() {
// Use chunk counts to speed things up if we can
if (this.localChunkCount != null && this.totalChunkCount != null &&
this.localChunkCount >= this.totalChunkCount) {
return true;
}
try {
this.fetchTransactions();
@ -165,6 +179,25 @@ public class ArbitraryDataResource {
}
}
private void calculateChunkCounts() {
try {
this.fetchTransactions();
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
int localChunkCount = 0;
int totalChunkCount = 0;
for (ArbitraryTransactionData transactionData : transactionDataList) {
localChunkCount += ArbitraryTransactionUtils.ourChunkCount(transactionData);
totalChunkCount += ArbitraryTransactionUtils.totalChunkCount(transactionData);
}
this.localChunkCount = localChunkCount;
this.totalChunkCount = totalChunkCount;
} catch (DataException e) {}
}
private boolean isRateLimited() {
try {
this.fetchTransactions();

10
src/main/java/org/qortal/data/arbitrary/ArbitraryResourceStatus.java
View File

@ -30,13 +30,21 @@ public class ArbitraryResourceStatus {
private String title;
private String description;
private Integer localChunkCount;
private Integer totalChunkCount;
public ArbitraryResourceStatus() {
}
public ArbitraryResourceStatus(Status status) {
public ArbitraryResourceStatus(Status status, Integer localChunkCount, Integer totalChunkCount) {
this.id = status.toString();
this.title = status.title;
this.description = status.description;
this.localChunkCount = localChunkCount;
this.totalChunkCount = totalChunkCount;
}
public ArbitraryResourceStatus(Status status) {
this(status, null, null);
}
}

40
src/main/java/org/qortal/utils/ArbitraryTransactionUtils.java
View File

@ -1,5 +1,6 @@
package org.qortal.utils;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataFile;
@ -147,16 +148,49 @@ public class ArbitraryTransactionUtils {
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata, therefore it has no chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
// Find the folder containing the files
Path parentPath = arbitraryDataFile.getFilePath().getParent();
String[] files = parentPath.toFile().list();
if (files == null) {
return 0;
}
// Remove the original copy indicator file if it exists
files = ArrayUtils.removeElement(files, ".original");
int count = files.length;
// If the complete file exists (and this transaction has chunks), subtract it from the count
if (arbitraryDataFile.chunkCount() > 0 && arbitraryDataFile.exists()) {
// We are only measuring the individual chunks, not the joined file
count -= 1;
}
return count;
}
public static int totalChunkCount(ArbitraryTransactionData transactionData) throws DataException {
if (transactionData == null) {
return 0;
}
byte[] digest = transactionData.getData();
byte[] metadataHash = transactionData.getMetadataHash();
byte[] signature = transactionData.getSignature();
if (metadataHash == null) {
// This file doesn't have any metadata, therefore it has a single (complete) chunk
return 1;
}
// Load complete file and chunks
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest, signature);
arbitraryDataFile.setMetadataHash(metadataHash);
return arbitraryDataFile.chunkCount();
return arbitraryDataFile.chunkCount() + 1; // +1 for the metadata file
}
public static boolean isFileRecent(Path filePath, long now, long cleanupAfter) {