crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Track the storage capacity and the total data/temp directory sizes

Browse Source 
Nodes will stop proactively storing new data when they reach 90% capacity.

A new "maxStorageCapacity" setting has been added to allow the user to optionally limit the allocated space for this node. Limits are approximate only, not exact.
This commit is contained in:
CalDescent 2021-11-24 13:43:45 +00:00
parent ae0f01d326
commit 8bf7daff65
5 changed files with 214 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/main/java/org/qortal/controller/Controller.java
View File

@ -54,6 +54,7 @@ import org.qortal.controller.arbitrary.ArbitraryDataBuildManager;
import org.qortal.controller.arbitrary.ArbitraryDataCleanupManager;
import org.qortal.controller.arbitrary.ArbitraryDataManager;
import org.qortal.controller.Synchronizer.SynchronizationResult;
import org.qortal.controller.arbitrary.ArbitraryDataStorageManager;
import org.qortal.controller.repository.PruneManager;
import org.qortal.controller.repository.NamesDatabaseIntegrityCheck;
import org.qortal.controller.tradebot.TradeBot;
@ -490,6 +491,7 @@ public class Controller extends Thread {
ArbitraryDataManager.getInstance().start();
ArbitraryDataBuildManager.getInstance().start();
ArbitraryDataCleanupManager.getInstance().start();
ArbitraryDataStorageManager.getInstance().start();
// Auto-update service?
if (Settings.getInstance().isAutoUpdateEnabled()) {
@ -1080,6 +1082,7 @@ public class Controller extends Thread {
ArbitraryDataManager.getInstance().shutdown();
ArbitraryDataBuildManager.getInstance().shutdown();
ArbitraryDataCleanupManager.getInstance().shutdown();
ArbitraryDataStorageManager.getInstance().shutdown();
if (blockMinter != null) {
LOGGER.info("Shutting down block minter");

5
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -76,6 +76,11 @@ public class ArbitraryDataCleanupManager extends Thread {
continue;
}
// Wait until storage capacity has been calculated
if (!storageManager.isStorageCapacityCalculated()) {
continue;
}
// Periodically delete any unnecessary files from the temp directory
if (offset == 0 || offset % (limit * 10) == 0) {
this.cleanupTempDirectory(now);

204
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -1,10 +1,20 @@
package org.qortal.controller.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.list.ResourceListManager;
import org.qortal.settings.Settings;
import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.NTP;
public class ArbitraryDataStorageManager {
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
public class ArbitraryDataStorageManager extends Thread {
public enum StoragePolicy {
FOLLOWED_AND_VIEWED,
@ -14,7 +24,21 @@ public class ArbitraryDataStorageManager {
NONE
}
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataStorageManager.class);
private static ArbitraryDataStorageManager instance;
private volatile boolean isStopping = false;
private Long storageCapacity = null;
private long totalDirectorySize = 0L;
private long lastDirectorySizeCheck = 0;
private static long DIRECTORY_SIZE_CHECK_INTERVAL = 10 * 60 * 1000L; // 10 minutes
/** Treat storage as full at 90% usage, to reduce risk of going over the limit.
* This is necessary because we don't calculate total storage values before every write.
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */
private static double STORAGE_FULL_THRESHOLD = 0.9; // 90%
public ArbitraryDataStorageManager() {
}
@ -26,6 +50,42 @@ public class ArbitraryDataStorageManager {
return instance;
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Storage Manager");
try {
while (!isStopping) {
Thread.sleep(1000);
Long now = NTP.getTime();
if (now == null) {
continue;
}
// Check the total directory size if we haven't in a while
if (this.shouldCalculateDirectorySize(now)) {
this.calculateDirectorySize(now);
}
Thread.sleep(59000);
}
} catch (InterruptedException e) {
// Fall-through to exit thread...
}
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
/**
* Check if data relating to a transaction is allowed to
* exist on this node, therefore making it a mirror for this data.
*
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean canStoreData(ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
@ -34,6 +94,8 @@ public class ArbitraryDataStorageManager {
return false;
}
// Don't check for storage limits here, as it can cause the cleanup manager to delete existing data
// Check if our storage policy and blacklist allows us to host data for this name
switch (Settings.getInstance().getStoragePolicy()) {
case FOLLOWED_AND_VIEWED:
@ -53,20 +115,38 @@ public class ArbitraryDataStorageManager {
}
}
/**
* Check if data relating to a transaction should be downloaded
* automatically, making this node a mirror for that data.
*
* @param arbitraryTransactionData - the transaction
* @return boolean - whether to prefetch or not
*/
public boolean shouldPreFetchData(ArbitraryTransactionData arbitraryTransactionData) {
String name = arbitraryTransactionData.getName();
if (name == null) {
return this.shouldPreFetchDataWithoutName(arbitraryTransactionData);
}
// Never fetch data from blacklisted names, even if they are followed
if (this.isNameInBlacklist(name)) {
// Don't fetch anything more if we're (nearly) out of space
// Make sure to keep STORAGE_FULL_THRESHOLD considerably less than 1, to
// avoid a fetch/delete loop
if (!this.isStorageSpaceAvailable(STORAGE_FULL_THRESHOLD)) {
return false;
}
// Don't store data unless it's an allowed type (public/private)
if (!this.isDataTypeAllowed(arbitraryTransactionData)) {
return false;
}
// Handle transactions without names differently
if (name == null) {
return this.shouldPreFetchDataWithoutName();
}
// Never fetch data from blacklisted names, even if they are followed
if (this.isNameInBlacklist(name)) {
return false;
}
switch (Settings.getInstance().getStoragePolicy()) {
case FOLLOWED:
case FOLLOWED_AND_VIEWED:
@ -82,10 +162,17 @@ public class ArbitraryDataStorageManager {
}
}
private boolean shouldPreFetchDataWithoutName(ArbitraryTransactionData arbitraryTransactionData) {
/**
* Don't call this method directly.
* Use the wrapper method shouldPreFetchData() instead, as it contains
* additional checks.
*
* @return boolean - whether the storage policy allows for unnamed data
*/
private boolean shouldPreFetchDataWithoutName() {
switch (Settings.getInstance().getStoragePolicy()) {
case ALL:
return this.isDataTypeAllowed(arbitraryTransactionData);
return true;
case NONE:
case VIEWED:
@ -118,4 +205,105 @@ public class ArbitraryDataStorageManager {
private boolean isFollowingName(String name) {
return ResourceListManager.getInstance().listContains("followed", "names", name, false);
}
// Size limits
/**
* Rate limit to reduce IO load
*/
private boolean shouldCalculateDirectorySize(Long now) {
if (now == null) {
return false;
}
// If storage capacity is null, we need to calculate it
if (this.storageCapacity == null) {
return true;
}
// If we haven't checked for a while, we need to check it now
if (now - lastDirectorySizeCheck > DIRECTORY_SIZE_CHECK_INTERVAL) {
return true;
}
// We shouldn't check this time, as we want to reduce IO load on the SSD/HDD
return false;
}
private void calculateDirectorySize(Long now) {
if (now == null) {
return;
}
long totalSize = 0;
long remainingCapacity = 0;
// Calculate remaining capacity
try {
remainingCapacity = this.getRemainingUsableStorageCapacity();
} catch (IOException e) {
LOGGER.info("Unable to calculate remaining storage capacity: {}", e.getMessage());
return;
}
// Calculate total size of data directory
LOGGER.trace("Calculating data directory size...");
Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
if (dataDirectoryPath.toFile().exists()) {
totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
}
// Add total size of temp directory, if it's not already inside the data directory
Path tempDirectoryPath = Paths.get(Settings.getInstance().getTempDataPath());
if (tempDirectoryPath.toFile().exists()) {
if (!FilesystemUtils.isChild(tempDirectoryPath, dataDirectoryPath)) {
LOGGER.trace("Calculating temp directory size...");
totalSize += FileUtils.sizeOfDirectory(dataDirectoryPath.toFile());
}
}
this.totalDirectorySize = totalSize;
this.lastDirectorySizeCheck = now;
// It's essential that used space (this.totalDirectorySize) is included in the storage capacity
LOGGER.trace("Calculating total storage capacity...");
long storageCapacity = remainingCapacity + this.totalDirectorySize;
// Make sure to limit the storage capacity if the user is overriding it in the settings
if (Settings.getInstance().getMaxStorageCapacity() != null) {
storageCapacity = Math.min(storageCapacity, Settings.getInstance().getMaxStorageCapacity());
}
this.storageCapacity = storageCapacity;
LOGGER.info("Total used: {} bytes, Total capacity: {} bytes", this.totalDirectorySize, this.storageCapacity);
}
private long getRemainingUsableStorageCapacity() throws IOException {
// Create data directory if it doesn't exist so that we can perform calculations on it
Path dataDirectoryPath = Paths.get(Settings.getInstance().getDataPath());
if (!dataDirectoryPath.toFile().exists()) {
Files.createDirectories(dataDirectoryPath);
}
return dataDirectoryPath.toFile().getUsableSpace();
}
public long getTotalDirectorySize() {
return this.totalDirectorySize;
}
public boolean isStorageSpaceAvailable(double threshold) {
if (!this.isStorageCapacityCalculated()) {
return false;
}
long maxStorageCapacity = (long)((double)this.storageCapacity / 100.0f * threshold);
if (this.totalDirectorySize >= maxStorageCapacity) {
return false;
}
return true;
}
public boolean isStorageCapacityCalculated() {
return (this.storageCapacity != null);
}
}

6
src/main/java/org/qortal/settings/Settings.java
View File

@ -289,6 +289,8 @@ public class Settings {
/** Whether to allow private (non-decryptable) data to be stored */
private boolean privateDataEnabled = false;
/** Maximum total size of hosted data, in bytes. Unlimited if null */
private Long maxStorageCapacity = null;
// Domain mapping
public static class DomainMap {
@ -840,4 +842,8 @@ public class Settings {
public boolean isPrivateDataEnabled() {
return this.privateDataEnabled;
}
public Long getMaxStorageCapacity() {
return this.maxStorageCapacity;
}
}

4
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@ -181,6 +181,10 @@ public class FilesystemUtils {
return false;
}
public static boolean isChild(Path child, Path parent) {
return child.toAbsolutePath().startsWith(parent.toAbsolutePath());
}
public static long getDirectorySize(Path path) throws IOException {
if (path == null || !Files.exists(path)) {
return 0L;