crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Delete directories in the data folder that have no associated transaction.

Browse Source 
This adds support for "unconfirmable" data uploads, which will be useful for Q-Chat. It also handles cases where a transaction is orphaned and then subsequently becomes invalid.
This commit is contained in:
CalDescent 2021-12-05 16:24:21 +00:00
parent c3835cefb1
commit ffe178c64c
2 changed files with 74 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java
View File

@ -204,6 +204,9 @@ public class ArbitraryDataCleanupManager extends Thread {
try (final Repository repository = RepositoryManager.getRepository()) {
// Check if there are any hosted files that don't have matching transactions
this.checkForExpiredTransactions(repository);
// Delete additional data at random if we're over our storage limit
// Use a threshold of 1, for the same reasons as above
if (!storageManager.isStorageSpaceAvailable(1.0f)) {
@ -233,6 +236,45 @@ public class ArbitraryDataCleanupManager extends Thread {
}
}
public List<Path> findPathsWithNoAssociatedTransaction(Repository repository) {
List<Path> pathList = new ArrayList<>();
// Find all hosted paths
List<Path> allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths();
// Loop through each path and find those without matching signatures
for (Path path : allPaths) {
try {
String[] contents = path.toFile().list();
if (contents == null || contents.length == 0) {
// Ignore empty directories
continue;
}
String signature58 = path.getFileName().toString();
byte[] signature = Base58.decode(signature58);
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null) {
// No transaction data, and no DataException, so we can assume that this data relates to an expired transaction
pathList.add(path);
}
} catch (DataException e) {
continue;
}
}
return pathList;
}
private void checkForExpiredTransactions(Repository repository) {
List<Path> expiredPaths = this.findPathsWithNoAssociatedTransaction(repository);
for (Path expiredPath : expiredPaths) {
LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString());
this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction");
}
}
private void storageLimitReached(Repository repository) throws InterruptedException {
// We think that the storage limit has been reached

50
src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java
View File

@ -244,28 +244,14 @@ public class ArbitraryDataStorageManager extends Thread {
List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
// Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
// Also exclude the _temp and _misc paths if present
List<Path> allPaths = new ArrayList<>();
try {
allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
}
catch (IOException e) {
LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
}
// Find all hosted paths
List<Path> allPaths = this.findAllHostedPaths();
// Loop through each path and attempt to match it to a signature
for (Path path : allPaths) {
try {
if (path.toFile().list().length == 0) {
String[] contents = path.toFile().list();
if (contents == null || contents.length == 0) {
// Ignore empty directories
continue;
}
@ -289,6 +275,34 @@ public class ArbitraryDataStorageManager extends Thread {
return arbitraryTransactionDataList;
}
/**
* Warning: this method will walk through the entire data directory
* Do not call it too frequently as it could create high disk load
* in environments with a large amount of hosted data.
* @return a list of paths that are being hosted
*/
public List<Path> findAllHostedPaths() {
Path dataPath = Paths.get(Settings.getInstance().getDataPath());
Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
// Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
// Also exclude the _temp and _misc paths if present
List<Path> allPaths = new ArrayList<>();
try {
allPaths = Files.walk(dataPath, 3)
.filter(Files::isDirectory)
.filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
&& !path.toString().contains("_misc")
&& path.getFileName().toString().length() > 32)
.collect(Collectors.toList());
}
catch (IOException e) {
LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
}
return allPaths;
}
public void invalidateHostedTransactionsCache() {
this.hostedTransactions = null;
}