Qortal/qortal
1
0
Fork 1
mirror of https://github.com/Qortal/qortal.git synced 2025-07-23 04:36:50 +00:00
Code Issues Packages Projects Releases Wiki Activity

Limit the amount of data that can be stored per name.

Browse Source 
This is calculated by the total capacity divided by the number of names the node follows. The idea here is that a single content creator can't upload terabytes of data and consume all the space on their followers' nodes. They can only use a proportion, with equal space given to each followed name. And since the limit is dynamic, following more names reduces the allocation to existing names.
This commit is contained in:
CalDescent
2021-12-04 13:33:45 +00:00
parent a87fe8b44d
commit a320bea68a
7 changed files with 270 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@@ -609,7 +609,7 @@ public class ArbitraryResource {
return hostedTransactions;
} catch (DataException | IOException e) {
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@@ -657,7 +657,7 @@ public class ArbitraryResource {
return resources;
} catch (DataException | IOException e) {
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@@ -190,7 +190,7 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
// Skip transactions that we don't need to proactively store data for
if (!storageManager.shouldPreFetchData(arbitraryTransactionData)) {
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
iterator.remove();
continue;
}

104
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@@ -20,6 +20,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class ArbitraryDataStorageManager extends Thread {
@@ -43,12 +44,12 @@ public class ArbitraryDataStorageManager extends Thread {
private List<ArbitraryTransactionData> hostedTransactions;
private static long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
/** Treat storage as full at 90% usage, to reduce risk of going over the limit.
* This is necessary because we don't calculate total storage values before every write.
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */
private static double STORAGE_FULL_THRESHOLD = 0.9; // 90%
private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90%
public ArbitraryDataStorageManager() {
}
@@ -133,7 +134,7 @@ public class ArbitraryDataStorageManager extends Thread {
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean shouldPreFetchData(ArbitraryTransactionData arbitraryTransactionData) {
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
// Don't fetch anything more if we're (nearly) out of space
@@ -143,6 +144,13 @@ public class ArbitraryDataStorageManager extends Thread {
return false;
}
// Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return false;
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false;
@@ -217,10 +225,14 @@ public class ArbitraryDataStorageManager extends Thread {
return ResourceListManager.getInstance().listContains("followed", "names", name, false);
}
private int followedNamesCount() {
return ResourceListManager.getInstance().getItemCountForList("followed", "names");
}
// Hosted data
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository) throws IOException {
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository) {
// Load from cache if we can, to avoid disk reads
if (this.hostedTransactions != null) {
return this.hostedTransactions;
@@ -233,12 +245,18 @@ public class ArbitraryDataStorageManager extends Thread {
// Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
// Also exclude the _temp and _misc paths if present
List<Path> allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
List<Path> allPaths = new ArrayList<>();
try {
allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
}
catch (IOException e) {
LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
}
// Loop through each path and attempt to match it to a signature
for (Path path : allPaths) {
@@ -277,7 +295,7 @@ public class ArbitraryDataStorageManager extends Thread {
/**
* Rate limit to reduce IO load
*/
private boolean shouldCalculateDirectorySize(Long now) {
public boolean shouldCalculateDirectorySize(Long now) {
if (now == null) {
return false;
}
@@ -368,7 +386,71 @@ public class ArbitraryDataStorageManager extends Thread {
return true;
}
public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
if (!this.isStorageSpaceAvailable(threshold)) {
// No storage space available at all, so no need to check this name
return false;
}
if (name == null) {
// This transaction doesn't have a name, so fall back to total space limitations
return true;
}
int followedNamesCount = this.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have space
return true;
}
long totalSizeForName = 0;
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {
// Transaction relates to a different name
continue;
}
totalSizeForName += transactionData.getSize();
}
// Have we reached the limit for this name?
if (totalSizeForName > maxStoragePerName) {
return false;
}
return true;
}
public long storageCapacityPerName(double threshold) {
int followedNamesCount = this.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have the total space available
return this.getStorageCapacityIncludingThreshold(threshold);
}
double maxStorageCapacity = (double)this.storageCapacity * threshold;
long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount);
return maxStoragePerName;
}
public boolean isStorageCapacityCalculated() {
return (this.storageCapacity != null);
}
public Long getStorageCapacity() {
return this.storageCapacity;
}
public Long getStorageCapacityIncludingThreshold(double threshold) {
if (this.storageCapacity == null) {
return null;
}
return (long)(this.storageCapacity * threshold);
}
}

8
src/main/java/org/qortal/list/ResourceListManager.java
View File

@@ -133,4 +133,12 @@ public class ResourceListManager {
return list.getList();
}
public int getItemCountForList(String category, String resourceName) {
ResourceList list = this.getList(category, resourceName);
if (list == null) {
return 0;
}
return list.getList().size();
}
}