Limit the amount of data that can be stored per name.

This is calculated by the total capacity divided by the number of names the node follows. The idea here is that a single content creator can't upload terabytes of data and consume all the space on their followers' nodes. They can only use a proportion, with equal space given to each followed name. And since the limit is dynamic, following more names reduces the allocation to existing names.

src/test/java/org/qortal/test/arbitrary/ArbitraryDataStorageCapacityTests.java

@@ -0,0 +1,134 @@
+package org.qortal.test.arbitrary;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
+import org.qortal.list.ResourceListManager;
+import org.qortal.repository.DataException;
+import org.qortal.settings.Settings;
+import org.qortal.test.common.Common;
+import org.qortal.utils.NTP;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import static org.junit.Assert.*;
+
+public class ArbitraryDataStorageCapacityTests extends Common {
+
+    @Before
+    public void beforeTest() throws DataException, InterruptedException {
+        Common.useDefaultSettings();
+        this.deleteDataDirectories();
+        this.deleteListsDirectory();
+    }
+
+    @After
+    public void afterTest() throws DataException {
+        this.deleteDataDirectories();
+        this.deleteListsDirectory();
+        ArbitraryDataStorageManager.getInstance().shutdown();
+    }
+
+
+    @Test
+    public void testCalculateTotalStorageCapacity() {
+        ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+        double storageFullThreshold = 0.9; // 90%
+        Long now = NTP.getTime();
+        assertNotNull("NTP time must be synced", now);
+        long expectedTotalStorageCapacity = Settings.getInstance().getMaxStorageCapacity();
+
+        // Capacity isn't initially calculated
+        assertNull(storageManager.getStorageCapacity());
+        assertEquals(0L, storageManager.getTotalDirectorySize());
+        assertFalse(storageManager.isStorageCapacityCalculated());
+
+        // We need to calculate the directory size because we haven't yet
+        assertTrue(storageManager.shouldCalculateDirectorySize(now));
+        storageManager.calculateDirectorySize(now);
+        assertTrue(storageManager.isStorageCapacityCalculated());
+
+        // Storage capacity should equal the value specified in settings
+        assertNotNull(storageManager.getStorageCapacity());
+        assertEquals(expectedTotalStorageCapacity, storageManager.getStorageCapacity().longValue());
+
+        // We shouldn't calculate storage capacity again so soon
+        now += 9 * 60 * 1000L;
+        assertFalse(storageManager.shouldCalculateDirectorySize(now));
+
+        // ... but after 10 minutes we should recalculate
+        now += 1 * 60 * 1000L + 1L;
+        assertTrue(storageManager.shouldCalculateDirectorySize(now));
+    }
+
+    @Test
+    public void testCalculateStorageCapacityPerName() {
+        ArbitraryDataStorageManager storageManager = ArbitraryDataStorageManager.getInstance();
+        ResourceListManager resourceListManager = ResourceListManager.getInstance();
+        double storageFullThreshold = 0.9; // 90%
+        Long now = NTP.getTime();
+        assertNotNull("NTP time must be synced", now);
+
+        // Capacity isn't initially calculated
+        assertNull(storageManager.getStorageCapacity());
+        assertEquals(0L, storageManager.getTotalDirectorySize());
+        assertFalse(storageManager.isStorageCapacityCalculated());
+
+        // We need to calculate the total directory size because we haven't yet
+        assertTrue(storageManager.shouldCalculateDirectorySize(now));
+        storageManager.calculateDirectorySize(now);
+        assertTrue(storageManager.isStorageCapacityCalculated());
+
+        // Storage capacity should initially equal the total
+        assertEquals(0, resourceListManager.getItemCountForList("followed", "names"));
+        long totalStorageCapacity = storageManager.getStorageCapacityIncludingThreshold(storageFullThreshold);
+        assertEquals(totalStorageCapacity, storageManager.storageCapacityPerName(storageFullThreshold));
+
+        // Follow some names
+        assertTrue(resourceListManager.addToList("followed", "names", "Test1", false));
+        assertTrue(resourceListManager.addToList("followed", "names", "Test2", false));
+        assertTrue(resourceListManager.addToList("followed", "names", "Test3", false));
+        assertTrue(resourceListManager.addToList("followed", "names", "Test4", false));
+
+        // Ensure the followed name count is correct
+        assertEquals(4, resourceListManager.getItemCountForList("followed", "names"));
+
+        // Storage space per name should be the total storage capacity divided by the number of names
+        long expectedStorageCapacityPerName = (long)(totalStorageCapacity / 4.0f);
+        assertEquals(expectedStorageCapacityPerName, storageManager.storageCapacityPerName(storageFullThreshold));
+    }
+
+
+    private void deleteDataDirectories() {
+        // Delete data directory if exists
+        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        try {
+            FileUtils.deleteDirectory(dataPath.toFile());
+        } catch (IOException e) {
+
+        }
+
+        // Delete temp data directory if exists
+        Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
+        try {
+            FileUtils.deleteDirectory(tempDataPath.toFile());
+        } catch (IOException e) {
+
+        }
+    }
+
+    private void deleteListsDirectory() {
+        // Delete lists directory if exists
+        Path listsPath = Paths.get(Settings.getInstance().getListsPath());
+        try {
+            FileUtils.deleteDirectory(listsPath.toFile());
+        } catch (IOException e) {
+
+        }
+    }
+
+}

src/test/java/org/qortal/test/arbitrary/ArbitraryDataStoragePolicyTests.java

@@ -34,6 +34,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
     @Before
     public void beforeTest() throws DataException, InterruptedException {
         Common.useDefaultSettings();
+        this.deleteDataDirectories();
         this.deleteListsDirectory();
         ArbitraryDataStorageManager.getInstance().start();
 
@@ -45,6 +46,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
 
     @After
     public void afterTest() throws DataException {
+        this.deleteDataDirectories();
         this.deleteListsDirectory();
         ArbitraryDataStorageManager.getInstance().shutdown();
     }
@@ -68,14 +70,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.FOLLOWED_AND_VIEWED, Settings.getInstance().getStoragePolicy());
             assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(transactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
 
             // We should store but not pre-fetch data for this transaction
             assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -101,14 +103,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.FOLLOWED, Settings.getInstance().getStoragePolicy());
             assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(transactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
 
             // We shouldn't store or pre-fetch data for this transaction
             assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -134,14 +136,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             // We should store but not pre-fetch data for this transaction
             assertEquals(StoragePolicy.VIEWED, Settings.getInstance().getStoragePolicy());
             assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
 
             // We should store but not pre-fetch data for this transaction
             assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -167,14 +169,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             // We should store and pre-fetch data for this transaction
             assertEquals(StoragePolicy.ALL, Settings.getInstance().getStoragePolicy());
             assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(transactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
 
             // We should store and pre-fetch data for this transaction
             assertTrue(storageManager.canStoreData(transactionData));
-            assertTrue(storageManager.shouldPreFetchData(transactionData));
+            assertTrue(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -200,14 +202,14 @@ public class ArbitraryDataStoragePolicyTests extends Common {
             // We shouldn't store or pre-fetch data for this transaction
             assertEquals(StoragePolicy.NONE, Settings.getInstance().getStoragePolicy());
             assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
 
             // Now unfollow the name
             assertTrue(ResourceListManager.getInstance().removeFromList("followed", "names", name, false));
 
             // We shouldn't store or pre-fetch data for this transaction
             assertFalse(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -223,7 +225,7 @@ public class ArbitraryDataStoragePolicyTests extends Common {
 
             // We should store but not pre-fetch data for this transaction
             assertTrue(storageManager.canStoreData(transactionData));
-            assertFalse(storageManager.shouldPreFetchData(transactionData));
+            assertFalse(storageManager.shouldPreFetchData(repository, transactionData));
         }
     }
 
@@ -240,6 +242,24 @@ public class ArbitraryDataStoragePolicyTests extends Common {
         return transactionData;
     }
 
+    private void deleteDataDirectories() {
+        // Delete data directory if exists
+        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        try {
+            FileUtils.deleteDirectory(dataPath.toFile());
+        } catch (IOException e) {
+
+        }
+
+        // Delete temp data directory if exists
+        Path tempDataPath = Paths.get(Settings.getInstance().getTempDataPath());
+        try {
+            FileUtils.deleteDirectory(tempDataPath.toFile());
+        } catch (IOException e) {
+
+        }
+    }
+
     private void deleteListsDirectory() {
         // Delete lists directory if exists
         Path listsPath = Paths.get(Settings.getInstance().getListsPath());

src/test/resources/test-settings-v2.json

@@ -15,5 +15,6 @@
   "tempDataPath": "data-test/_temp",
   "listsPath": "lists-test",
   "storagePolicy": "FOLLOWED_AND_VIEWED",
+  "maxStorageCapacity": 104857600,
   "localAuthBypassEnabled": true
 }