crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Limit the amount of data that can be stored per name.

Browse Source 
This is calculated by the total capacity divided by the number of names the node follows. The idea here is that a single content creator can't upload terabytes of data and consume all the space on their followers' nodes. They can only use a proportion, with equal space given to each followed name. And since the limit is dynamic, following more names reduces the allocation to existing names.
This commit is contained in:
CalDescent 2021-12-04 13:33:45 +00:00
parent a87fe8b44d
commit a320bea68a
7 changed files with 270 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -609,7 +609,7 @@ public class ArbitraryResource {
return hostedTransactions;
} catch (DataException | IOException e) {
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}
@ -657,7 +657,7 @@ public class ArbitraryResource {
return resources;
} catch (DataException | IOException e) {
} catch (DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
}
}

2
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataManager.java
View File

@ -190,7 +190,7 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) arbitraryTransaction.getTransactionData();
// Skip transactions that we don't need to proactively store data for
if (!storageManager.shouldPreFetchData(arbitraryTransactionData)) {
if (!storageManager.shouldPreFetchData(repository, arbitraryTransactionData)) {
iterator.remove();
continue;
}

104
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -20,6 +20,7 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.stream.Collectors;
public class ArbitraryDataStorageManager extends Thread {
@ -43,12 +44,12 @@ public class ArbitraryDataStorageManager extends Thread {
private List<ArbitraryTransactionData> hostedTransactions;
private static long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
private static final long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
/** Treat storage as full at 90% usage, to reduce risk of going over the limit.
* This is necessary because we don't calculate total storage values before every write.
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */
private static double STORAGE_FULL_THRESHOLD = 0.9; // 90%
private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90%
public ArbitraryDataStorageManager() {
}
@ -133,7 +134,7 @@ public class ArbitraryDataStorageManager extends Thread {
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean shouldPreFetchData(ArbitraryTransactionData arbitraryTransactionData) {
public boolean shouldPreFetchData(Repository repository, ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
// Don't fetch anything more if we're (nearly) out of space
@ -143,6 +144,13 @@ public class ArbitraryDataStorageManager extends Thread {
return false;
}
// Don't fetch anything if we're (nearly) out of space for this name
// Again, make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailableForName(repository, arbitraryTransactionData.getName(), STORAGE_FULL_THRESHOLD)) {
return false;
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false;
@ -217,10 +225,14 @@ public class ArbitraryDataStorageManager extends Thread {
return ResourceListManager.getInstance().listContains("followed", "names", name, false);
}
private int followedNamesCount() {
return ResourceListManager.getInstance().getItemCountForList("followed", "names");
}
// Hosted data
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository) throws IOException {
public List<ArbitraryTransactionData> listAllHostedTransactions(Repository repository) {
// Load from cache if we can, to avoid disk reads
if (this.hostedTransactions != null) {
return this.hostedTransactions;
@ -233,12 +245,18 @@ public class ArbitraryDataStorageManager extends Thread {
// Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
// Also exclude the _temp and _misc paths if present
List<Path> allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
List<Path> allPaths = new ArrayList<>();
try {
allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
}
catch (IOException e) {
LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
}
// Loop through each path and attempt to match it to a signature
for (Path path : allPaths) {
@ -277,7 +295,7 @@ public class ArbitraryDataStorageManager extends Thread {
/**
* Rate limit to reduce IO load
*/
private boolean shouldCalculateDirectorySize(Long now) {
public boolean shouldCalculateDirectorySize(Long now) {
if (now == null) {
return false;
}
@ -368,7 +386,71 @@ public class ArbitraryDataStorageManager extends Thread {
return true;
}
public boolean isStorageSpaceAvailableForName(Repository repository, String name, double threshold) {
if (!this.isStorageSpaceAvailable(threshold)) {
// No storage space available at all, so no need to check this name
return false;
}
if (name == null) {
// This transaction doesn't have a name, so fall back to total space limitations
return true;
}
int followedNamesCount = this.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have space
return true;
}
long totalSizeForName = 0;
long maxStoragePerName = this.storageCapacityPerName(threshold);
// Fetch all hosted transactions
List<ArbitraryTransactionData> hostedTransactions = this.listAllHostedTransactions(repository);
for (ArbitraryTransactionData transactionData : hostedTransactions) {
String transactionName = transactionData.getName();
if (!Objects.equals(name, transactionName)) {
// Transaction relates to a different name
continue;
}
totalSizeForName += transactionData.getSize();
}
// Have we reached the limit for this name?
if (totalSizeForName > maxStoragePerName) {
return false;
}
return true;
}
public long storageCapacityPerName(double threshold) {
int followedNamesCount = this.followedNamesCount();
if (followedNamesCount == 0) {
// Not following any names, so we have the total space available
return this.getStorageCapacityIncludingThreshold(threshold);
}
double maxStorageCapacity = (double)this.storageCapacity * threshold;
long maxStoragePerName = (long)(maxStorageCapacity / (double)followedNamesCount);
return maxStoragePerName;
}
public boolean isStorageCapacityCalculated() {
return (this.storageCapacity != null);
}
public Long getStorageCapacity() {
return this.storageCapacity;
}
public Long getStorageCapacityIncludingThreshold(double threshold) {
if (this.storageCapacity == null) {
return null;
}
return (long)(this.storageCapacity * threshold);
}
}

8
src/main/java/org/qortal/list/ResourceListManager.java
View File

@ -133,4 +133,12 @@ public class ResourceListManager {
return list.getList();
}
public int getItemCountForList(String category, String resourceName) {
ResourceList list = this.getList(category, resourceName);
if (list == null) {
return 0;
}
return list.getList().size();
}
}

134
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java Normal file
View File

@ -0,0 +1,134 @@
package org.qortal.test.arbitrary;
import org.apache.commons.io.FileUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.list.ResourceListManager;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import org.qortal.test.common.Common;
import org.qortal.utils.NTP;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
import static org.junit.Assert.*;
public class ArbitraryDataStorageCapacityTests extends Common {
@Before
public void beforeTest() throws DataException, InterruptedException {
Common.useDefaultSettings();
this.deleteDataDirectories();
this.deleteListsDirectory();
}
@After
public void afterTest() throws DataException {
this.deleteDataDirectories();
this.deleteListsDirectory();
ArbitraryDataStorageManager.getInstance().shutdown();
}
@Test
public void testCalculateTotalStorageCapacity() {
ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
double storageFullThreshold = 0.9; // 90%
Long now = NTP.getTime();
assertNotNull("NTP time must be synced", now);
long expectedTotalStorageCapacity = Settings.getInstance().getMaxStorageCapacity();
// Capacity isn't initially calculated
assertNull(storageManager.getStorageCapacity());
assertEquals(0L, storageManager.getTotalDirectorySize());
assertFalse(storageManager.isStorageCapacityCalculated());
// We need to calculate the directory size because we haven't yet
assertTrue(storageManager.shouldCalculateDirectorySize(now));
storageManager.calculateDirectorySize(now);
assertTrue(storageManager.isStorageCapacityCalculated());
// Storage capacity should equal the value specified in settings
assertNotNull(storageManager.getStorageCapacity());
assertEquals(expectedTotalStorageCapacity, storageManager.getStorageCapacity().longValue());
// We shouldn't calculate storage capacity again so soon
now += 9 * 60 * 1000L;
assertFalse(storageManager.shouldCalculateDirectorySize(now));
// ... but after 10 minutes we should recalculate
now += 1 * 60 * 1000L + 1L;
assertTrue(storageManager.shouldCalculateDirectorySize(now));
}
@Test
public void testCalculateStorageCapacityPerName() {
ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
ResourceListManager resourceListManager = ResourceListManager.getInstance();
double storageFullThreshold = 0.9; // 90%
Long now = NTP.getTime();
assertNotNull("NTP time must be synced", now);
// Capacity isn't initially calculated
assertNull(storageManager.getStorageCapacity());
assertEquals(0L, storageManager.getTotalDirectorySize());
assertFalse(storageManager.isStorageCapacityCalculated());
// We need to calculate the total directory size because we haven't yet
assertTrue(storageManager.shouldCalculateDirectorySize(now));
storageManager.calculateDirectorySize(now);
assertTrue(storageManager.isStorageCapacityCalculated());
// Storage capacity should initially equal the total
assertEquals(0, resourceListManager.getItemCountForList("followed", "names"));
long totalStorageCapacity = storageManager.getStorageCapacityIncludingThreshold(storageFullThreshold);
assertEquals(totalStorageCapacity, storageManager.storageCapacityPerName(storageFullThreshold));
// Follow some names
assertTrue(resourceListManager.addToList("followed", "names", "Test1", false));
assertTrue(resourceListManager.addToList("followed", "names", "Test2", false));
assertTrue(resourceListManager.addToList("followed", "names", "Test3", false));
assertTrue(resourceListManager.addToList("followed", "names", "Test4", false));
// Ensure the followed name count is correct
assertEquals(4, resourceListManager.getItemCountForList("followed", "names"));
// Storage space per name should be the total storage capacity divided by the number of names
long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f);
assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold));
}
private void deleteDataDirectories() {
// Delete data directory if exists
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
try {
FileUtils.deleteDirectory(dataPath.toFile());
} catch (IOException e) {
}
// Delete temp data directory if exists
Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
try {
FileUtils.deleteDirectory(tempDataPath.toFile());
} catch (IOException e) {
}
}
private void deleteListsDirectory() {
// Delete lists directory if exists
Path listsPath = Paths.get(Settings.getInstance().getListsPath());
try {
FileUtils.deleteDirectory(listsPath.toFile());
} catch (IOException e) {
}
}
}

42
src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java
View File

@ -34,6 +34,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
@Before
public void beforeTest() throws DataException, InterruptedException {
Common.useDefaultSettings();
this.deleteDataDirectories();
this.deleteListsDirectory();
ArbitraryDataStorageManager.getInstance().start();
@ -45,6 +46,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
@After
public void afterTest() throws DataException {
this.deleteDataDirectories();
this.deleteListsDirectory();
ArbitraryDataStorageManager.getInstance().shutdown();
}
@ -68,14 +70,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED_AND_VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -101,14 +103,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -134,14 +136,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -167,14 +169,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store and pre-fetch data for this transaction
assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
// We should store and pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertTrue(storageManager.shouldPreFetchData(transactionData));
assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -200,14 +202,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We shouldn't store or pre-fetch data for this transaction
assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
// Now unfollow the name
assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
// We shouldn't store or pre-fetch data for this transaction
assertFalse(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -223,7 +225,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
// We should store but not pre-fetch data for this transaction
assertTrue(storageManager.canStoreData(transactionData));
assertFalse(storageManager.shouldPreFetchData(transactionData));
assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
}
}
@ -240,6 +242,24 @@ public class ArbitraryDataStoragePolicyTests extends Common {
return transactionData;
}
private void deleteDataDirectories() {
// Delete data directory if exists
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
try {
FileUtils.deleteDirectory(dataPath.toFile());
} catch (IOException e) {
}
// Delete temp data directory if exists
Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
try {
FileUtils.deleteDirectory(tempDataPath.toFile());
} catch (IOException e) {
}
}
private void deleteListsDirectory() {
// Delete lists directory if exists
Path listsPath = Paths.get(Settings.getInstance().getListsPath());

1
src/test/resources/test-settings-v2.json
View File

@ -15,5 +15,6 @@
"tempDataPath": "data-test/_temp",
"listsPath": "lists-test",
"storagePolicy": "FOLLOWED_AND_VIEWED",
"maxStorageCapacity": 104857600,
"localAuthBypassEnabled": true
}