From 8bf7daff653ebfd64eff893282d8c755ac3fc99b Mon Sep 17 00:00:00 2001 From: CalDescent Date: Wed, 24 Nov 2021 13:43:45 +0000 Subject: [PATCH] Track the storage capacity and the total data/temp directory sizes Nodes will stop proactively storing new data when they reach 90% capacity. A new "maxStorageCapacity" setting has been added to allow the user to optionally limit the allocated space for this node. Limits are approximate only, not exact. --- .../org/qortal/controller/Controller.java | 3 + .../ArbitraryDataCleanupManager.java | 5 + .../ArbitraryDataStorageManager.java | 204 +++++++++++++++++- .../java/org/qortal/settings/Settings.java | 6 + .../org/qortal/utils/FilesystemUtils.java | 4 + 5 files changed, 214 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index 1609dfa8..08be63ec 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -54,6 +54,7 @@ import org.qortal.controller.arbitrary.ArbitraryDataBuildManager; import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager; import org.qortal.controller.arbitrary.ArbitraryDataManager; import org.qortal.controller.Synchronizer.SynchronizationResult; +import org.qortal.controller.arbitrary.ArbitraryDataStorageManager; import org.qortal.controller.repository.PruneManager; import org.qortal.controller.repository.NamesDatabaseIntegrityCheck; import org.qortal.controller.tradebot.TradeBot; @@ -490,6 +491,7 @@ public class Controller extends Thread { ArbitraryDataManager.getInstance().start(); ArbitraryDataBuildManager.getInstance().start(); ArbitraryDataCleanupManager.getInstance().start(); + ArbitraryDataStorageManager.getInstance().start(); // Auto-update service? if (Settings.getInstance().isAutoUpdateEnabled()) { @@ -1080,6 +1082,7 @@ public class Controller extends Thread { ArbitraryDataManager.getInstance().shutdown(); ArbitraryDataBuildManager.getInstance().shutdown(); ArbitraryDataCleanupManager.getInstance().shutdown(); + ArbitraryDataStorageManager.getInstance().shutdown(); if (blockMinter != null) { LOGGER.info("Shutting down block minter"); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index 3d19bd96..a0e332e9 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -76,6 +76,11 @@ public class ArbitraryDataCleanupManager extends Thread { continue; } + // Wait until storage capacity has been calculated + if (!storageManager.isStorageCapacityCalculated()) { + continue; + } + // Periodically delete any unnecessary files from the temp directory if (offset == 0 || offset % (limit * 10) == 0) { this.cleanupTempDirectory(now); diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index 34eb94ba..42b176b6 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -1,10 +1,20 @@ package org.qortal.controller.arbitrary; +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.qortal.data.transaction.ArbitraryTransactionData; import org.qortal.list.ResourceListManager; import org.qortal.settings.Settings; +import org.qortal.utils.FilesystemUtils; +import org.qortal.utils.NTP; -public class ArbitraryDataStorageManager { +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +public class ArbitraryDataStorageManager extends Thread { public enum StoragePolicy { FOLLOWED_AND_VIEWED, @@ -14,7 +24,21 @@ public class ArbitraryDataStorageManager { NONE } + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataStorageManager.class); + private static ArbitraryDataStorageManager instance; + private volatile boolean isStopping = false; + + private Long storageCapacity = null; + private long totalDirectorySize = 0L; + private long lastDirectorySizeCheck = 0; + + private static long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes + + /** Treat storage as full at 90% usage, to reduce risk of going over the limit. + * This is necessary because we don't calculate total storage values before every write. + * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */ + private static double STORAGE_FULL_THRESHOLD = 0.9; // 90% public ArbitraryDataStorageManager() { } @@ -26,6 +50,42 @@ public class ArbitraryDataStorageManager { return instance; } + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Storage Manager"); + try { + while (!isStopping) { + Thread.sleep(1000); + + Long now = NTP.getTime(); + if (now == null) { + continue; + } + + // Check the total directory size if we haven't in a while + if (this.shouldCalculateDirectorySize(now)) { + this.calculateDirectorySize(now); + } + + Thread.sleep(59000); + } + } catch (InterruptedException e) { + // Fall-through to exit thread... + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + /** + * Check if data relating to a transaction is allowed to + * exist on this node, therefore making it a mirror for this data. + * + * @param arbitraryTransactionData - the transaction + * @return boolean - whether to prefetch or not + */ public boolean canStoreData(ArbitraryTransactionData arbitraryTransactionData) { String name = arbitraryTransactionData.getName(); @@ -34,6 +94,8 @@ public class ArbitraryDataStorageManager { return false; } + // Don't check for storage limits here, as it can cause the cleanup manager to delete existing data + // Check if our storage policy and blacklist allows us to host data for this name switch (Settings.getInstance().getStoragePolicy()) { case FOLLOWED_AND_VIEWED: @@ -53,20 +115,38 @@ public class ArbitraryDataStorageManager { } } + /** + * Check if data relating to a transaction should be downloaded + * automatically, making this node a mirror for that data. + * + * @param arbitraryTransactionData - the transaction + * @return boolean - whether to prefetch or not + */ public boolean shouldPreFetchData(ArbitraryTransactionData arbitraryTransactionData) { String name = arbitraryTransactionData.getName(); - if (name == null) { - return this.shouldPreFetchDataWithoutName(arbitraryTransactionData); - } - // Never fetch data from blacklisted names, even if they are followed - if (this.isNameInBlacklist(name)) { + + // Don't fetch anything more if we're (nearly) out of space + // Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to + // avoid a fetch/delete loop + if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) { return false; } + // Don't store data unless it's an allowed type (public/private) if (!this.isDataTypeAllowed(arbitraryTransactionData)) { return false; } + // Handle transactions without names differently + if (name == null) { + return this.shouldPreFetchDataWithoutName(); + } + + // Never fetch data from blacklisted names, even if they are followed + if (this.isNameInBlacklist(name)) { + return false; + } + switch (Settings.getInstance().getStoragePolicy()) { case FOLLOWED: case FOLLOWED_AND_VIEWED: @@ -82,10 +162,17 @@ public class ArbitraryDataStorageManager { } } - private boolean shouldPreFetchDataWithoutName(ArbitraryTransactionData arbitraryTransactionData) { + /** + * Don't call this method directly. + * Use the wrapper method shouldPreFetchData() instead, as it contains + * additional checks. + * + * @return boolean - whether the storage policy allows for unnamed data + */ + private boolean shouldPreFetchDataWithoutName() { switch (Settings.getInstance().getStoragePolicy()) { case ALL: - return this.isDataTypeAllowed(arbitraryTransactionData); + return true; case NONE: case VIEWED: @@ -118,4 +205,105 @@ public class ArbitraryDataStorageManager { private boolean isFollowingName(String name) { return ResourceListManager.getInstance().listContains("followed", "names", name, false); } + + + // Size limits + + /** + * Rate limit to reduce IO load + */ + private boolean shouldCalculateDirectorySize(Long now) { + if (now == null) { + return false; + } + // If storage capacity is null, we need to calculate it + if (this.storageCapacity == null) { + return true; + } + // If we haven't checked for a while, we need to check it now + if (now - lastDirectorySizeCheck > DIRECTORY_SIZE_CHECK_INTERVAL) { + return true; + } + + // We shouldn't check this time, as we want to reduce IO load on the SSD/HDD + return false; + } + + private void calculateDirectorySize(Long now) { + if (now == null) { + return; + } + + long totalSize = 0; + long remainingCapacity = 0; + + // Calculate remaining capacity + try { + remainingCapacity = this.getRemainingUsableStorageCapacity(); + } catch (IOException e) { + LOGGER.info("Unable to calculate remaining storage capacity: {}", e.getMessage()); + return; + } + + // Calculate total size of data directory + LOGGER.trace("Calculating data directory size..."); + Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath()); + if (dataDirectoryPath.toFile().exists()) { + totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile()); + } + + // Add total size of temp directory, if it's not already inside the data directory + Path tempDirectoryPath = Paths.get(Settings.getInstance().getTempDataPath()); + if (tempDirectoryPath.toFile().exists()) { + if (!FilesystemUtils.isChild(tempDirectoryPath, dataDirectoryPath)) { + LOGGER.trace("Calculating temp directory size..."); + totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile()); + } + } + + this.totalDirectorySize = totalSize; + this.lastDirectorySizeCheck = now; + + // It's essential that used space (this.totalDirectorySize) is included in the storage capacity + LOGGER.trace("Calculating total storage capacity..."); + long storageCapacity = remainingCapacity + this.totalDirectorySize; + + // Make sure to limit the storage capacity if the user is overriding it in the settings + if (Settings.getInstance().getMaxStorageCapacity() != null) { + storageCapacity = Math.min(storageCapacity, Settings.getInstance().getMaxStorageCapacity()); + } + this.storageCapacity = storageCapacity; + + LOGGER.info("Total used: {} bytes, Total capacity: {} bytes", this.totalDirectorySize, this.storageCapacity); + } + + private long getRemainingUsableStorageCapacity() throws IOException { + // Create data directory if it doesn't exist so that we can perform calculations on it + Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath()); + if (!dataDirectoryPath.toFile().exists()) { + Files.createDirectories(dataDirectoryPath); + } + + return dataDirectoryPath.toFile().getUsableSpace(); + } + + public long getTotalDirectorySize() { + return this.totalDirectorySize; + } + + public boolean isStorageSpaceAvailable(double threshold) { + if (!this.isStorageCapacityCalculated()) { + return false; + } + + long maxStorageCapacity = (long)((double)this.storageCapacity / 100.0f * threshold); + if (this.totalDirectorySize >= maxStorageCapacity) { + return false; + } + return true; + } + + public boolean isStorageCapacityCalculated() { + return (this.storageCapacity != null); + } } diff --git a/src/main/java/org/qortal/settings/Settings.java b/src/main/java/org/qortal/settings/Settings.java index 81dbb624..0894af5d 100644 --- a/src/main/java/org/qortal/settings/Settings.java +++ b/src/main/java/org/qortal/settings/Settings.java @@ -289,6 +289,8 @@ public class Settings { /** Whether to allow private (non-decryptable) data to be stored */ private boolean privateDataEnabled = false; + /** Maximum total size of hosted data, in bytes. Unlimited if null */ + private Long maxStorageCapacity = null; // Domain mapping public static class DomainMap { @@ -840,4 +842,8 @@ public class Settings { public boolean isPrivateDataEnabled() { return this.privateDataEnabled; } + + public Long getMaxStorageCapacity() { + return this.maxStorageCapacity; + } } diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index 0f2c16a4..c0375d99 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -181,6 +181,10 @@ public class FilesystemUtils { return false; } + public static boolean isChild(Path child, Path parent) { + return child.toAbsolutePath().startsWith(parent.toAbsolutePath()); + } + public static long getDirectorySize(Path path) throws IOException { if (path == null || !Files.exists(path)) { return 0L;