AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Tidy up of storage thresholds and comments. Also reduced deletion threshold from 100% to 98% to reduce the chances of filling up the disk and corrupting the db.

Browse Source
This commit is contained in:
CalDescent 2021-12-05 16:36:54 +00:00
parent 007f567c7a
commit c6e5c4e3b5
2 changed files with 18 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -23,6 +23,8 @@ import java.nio.file.Paths;
import java.security.SecureRandom;
import java.util.*;
import static org.qortal.controller.arbitrary.ArbitraryDataStorageManager.DELETION_THRESHOLD;
public class ArbitraryDataCleanupManager extends Thread {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataCleanupManager.class);
@ -51,7 +53,7 @@ public class ArbitraryDataCleanupManager extends Thread {
/*
TODO:
- Delete old files not associated with transactions
- Delete files from the _misc folder once they reach a certain age
*/
@ -208,8 +210,10 @@ public class ArbitraryDataCleanupManager extends Thread {
this.checkForExpiredTransactions(repository);
// Delete additional data at random if we're over our storage limit
// Use a threshold of 1, for the same reasons as above
if (!storageManager.isStorageSpaceAvailable(1.0f)) {
// Use the DELETION_THRESHOLD so that we only start deleting once the hard limit is reached
// This also allows some headroom between the regular threshold (90%) and the hard
// limit, to avoid data getting into a fetch/delete loop.
if (!storageManager.isStorageSpaceAvailable(DELETION_THRESHOLD)) {
// Rate limit, to avoid repeated calls to calculateDirectorySize()
Thread.sleep(60000);
@ -218,11 +222,9 @@ public class ArbitraryDataCleanupManager extends Thread {
}
// Delete random data associated with name if we're over our storage limit for this name
// Use a threshold of 1 so that we only start deleting once the hard limit is reached
// This also allows some headroom between the regular threshold (90%) and the hard
// limit, to avoid data getting into a fetch/delete loop.
// Use the DELETION_THRESHOLD, for the same reasons as above
for (String followedName : storageManager.followedNames()) {
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, 1.0f)) {
if (!storageManager.isStorageSpaceAvailableForName(repository, followedName, DELETION_THRESHOLD)) {
this.storageLimitReachedForName(repository, followedName);
}
}
@ -281,7 +283,7 @@ public class ArbitraryDataCleanupManager extends Thread {
// Now calculate the used/total storage again, as a safety precaution
Long now = NTP.getTime();
ArbitraryDataStorageManager.getInstance().calculateDirectorySize(now);
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(1.0f)) {
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailable(DELETION_THRESHOLD)) {
// We have space available, so don't delete anything
return;
}
@ -299,7 +301,7 @@ public class ArbitraryDataCleanupManager extends Thread {
public void storageLimitReachedForName(Repository repository, String name) throws InterruptedException {
// We think that the storage limit has been reached for supplied name - but we should double check
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, 1.0f)) {
if (ArbitraryDataStorageManager.getInstance().isStorageSpaceAvailableForName(repository, name, DELETION_THRESHOLD)) {
// We have space available for this name, so don't delete anything
return;
}

9
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -48,8 +48,13 @@ public class ArbitraryDataStorageManager extends Thread {
/** Treat storage as full at 90% usage, to reduce risk of going over the limit.
* This is necessary because we don't calculate total storage values before every write.
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit. */
private static final double STORAGE_FULL_THRESHOLD = 0.9; // 90%
* It also helps avoid a fetch/delete loop, as we will stop fetching before the hard limit.
* This must be lower than DELETION_THRESHOLD. */
private static final double STORAGE_FULL_THRESHOLD = 0.90f; // 90%
/** Start deleting files once we reach 98% usage.
* This must be higher than STORAGE_FULL_THRESHOLD in order to avoid a fetch/delete loop. */
public static final double DELETION_THRESHOLD = 0.98f; // 98%
public ArbitraryDataStorageManager() {
}