AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Tidy up of storage thresholds and comments. Also reduced deletion threshold from 100% to 98% to reduce the chances of filling up the disk and corrupting the db.

Browse Source
This commit is contained in:
CalDescent 2021-12-05 16:36:54 +00:00
parent 007f567c7a
commit c6e5c4e3b5
2 changed files with 18 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -23,6 +23,8 @@ import java.nio.file.Paths;
import java.security.SecureRandom; import java.security.SecureRandom;
import java.util.*; import java.util.*;
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
public class ArbitraryDataCleanupManager extends Thread { public class ArbitraryDataCleanupManager extends Thread {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class); private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class);
@ -51,7 +53,7 @@ public class ArbitraryDataCleanupManager extends Thread {
/* /*
TODO: TODO:
- Delete old files not associated with transactions - Delete files from the _misc folder once they reach a certain age
*/ */
@ -208,8 +210,10 @@ public class ArbitraryDataCleanupManager extends Thread {
this.checkForExpiredTransactions(repository); this.checkForExpiredTransactions(repository);
// Delete additional data at random if we're over our storage limit // Delete additional data at random if we're over our storage limit
// Use a threshold of 1, for the same reasons as above // Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached
if (!storageManager.isStorageSpaceAvailable(1.0f)) { // This also allows some headroom between the regular threshold (90%) and the hard
// limit, to avoid data getting into a fetch/delete loop.
if (!storageManager.isStorageSpaceAvailable(DELETION_THRESHOLD)) {
// Rate limit, to avoid repeated calls to calculateDirectorySize() // Rate limit, to avoid repeated calls to calculateDirectorySize()
Thread.sleep(60000); Thread.sleep(60000);
@ -218,11 +222,9 @@ public class ArbitraryDataCleanupManager extends Thread {
} }
// Delete random data associated with name if we're over our storage limit for this name // Delete random data associated with name if we're over our storage limit for this name
// Use a threshold of 1 so that we only start deleting once the hard limit is reached // Use the DELETION_THRESHOLD, for the same reasons as above
// This also allows some headroom between the regular threshold (90%) and the hard
// limit, to avoid data getting into a fetch/delete loop.
for (String followedName : storageManager.followedNames()) { for (String followedName : storageManager.followedNames()) {
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, 1.0f)) { if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName); this.storageLimitReachedForName(repository, followedName);
} }
} }
@ -281,7 +283,7 @@ public class ArbitraryDataCleanupManager extends Thread {
// Now calculate the used/total storage again, as a safety precaution // Now calculate the used/total storage again, as a safety precaution
Long now = NTP.getTime(); Long now = NTP.getTime();
ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now); ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now);
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(1.0f)) { if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) {
// We have space available, so don't delete anything // We have space available, so don't delete anything
return; return;
} }
@ -299,7 +301,7 @@ public class ArbitraryDataCleanupManager extends Thread {
public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException { public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
// We think that the storage limit has been reached for supplied name - but we should double check // We think that the storage limit has been reached for supplied name - but we should double check
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, 1.0f)) { if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
// We have space available for this name, so don't delete anything // We have space available for this name, so don't delete anything
return; return;
} }

9
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -48,8 +48,13 @@ public class ArbitraryDataStorageManager extends Thread {
/** Treat storage as full at 90% usage, to reduce risk of going over the limit. /** Treat storage as full at 90% usage, to reduce risk of going over the limit.
* This is necessary because we don't calculate total storage values before every write. * This is necessary because we don't calculate total storage values before every write.
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */ * It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90% * This must be lower than DELETION_THRESHOLD. */
private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
/** Start deleting files once we reach 98% usage.
* This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
public static final double DELETION_THRESHOLD = 0.98f; // 98%
public ArbitraryDataStorageManager() { public ArbitraryDataStorageManager() {
} }