diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java index eb114fdb..768b3344 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java @@ -204,6 +204,9 @@ public class ArbitraryDataCleanupManager extends Thread { try (final Repository repository = RepositoryManager.getRepository()) { + // Check if there are any hosted files that don't have matching transactions + this.checkForExpiredTransactions(repository); + // Delete additional data at random if we're over our storage limit // Use a threshold of 1, for the same reasons as above if (!storageManager.isStorageSpaceAvailable(1.0f)) { @@ -233,6 +236,45 @@ public class ArbitraryDataCleanupManager extends Thread { } } + public List findPathsWithNoAssociatedTransaction(Repository repository) { + List pathList = new ArrayList<>(); + + // Find all hosted paths + List allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths(); + + // Loop through each path and find those without matching signatures + for (Path path : allPaths) { + try { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { + // Ignore empty directories + continue; + } + + String signature58 = path.getFileName().toString(); + byte[] signature = Base58.decode(signature58); + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null) { + // No transaction data, and no DataException, so we can assume that this data relates to an expired transaction + pathList.add(path); + } + + } catch (DataException e) { + continue; + } + } + + return pathList; + } + + private void checkForExpiredTransactions(Repository repository) { + List expiredPaths = this.findPathsWithNoAssociatedTransaction(repository); + for (Path expiredPath : expiredPaths) { + LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString()); + this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction"); + } + } + private void storageLimitReached(Repository repository) throws InterruptedException { // We think that the storage limit has been reached diff --git a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java index b36770b6..f8195e0d 100644 --- a/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java +++ b/src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java @@ -244,28 +244,14 @@ public class ArbitraryDataStorageManager extends Thread { List arbitraryTransactionDataList = new ArrayList<>(); - Path dataPath = Paths.get(Settings.getInstance().getDataPath()); - Path tempPath = Paths.get(Settings.getInstance().getTempDataPath()); - - // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length - // Also exclude the _temp and _misc paths if present - List allPaths = new ArrayList<>(); - try { - allPaths = Files.walk(dataPath, 3) - .filter(Files::isDirectory) - .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()) - && !path.toString().contains("_misc") - && path.getFileName().toString().length() > 32) - .collect(Collectors.toList()); - } - catch (IOException e) { - LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); - } + // Find all hosted paths + List allPaths = this.findAllHostedPaths(); // Loop through each path and attempt to match it to a signature for (Path path : allPaths) { try { - if (path.toFile().list().length == 0) { + String[] contents = path.toFile().list(); + if (contents == null || contents.length == 0) { // Ignore empty directories continue; } @@ -289,6 +275,34 @@ public class ArbitraryDataStorageManager extends Thread { return arbitraryTransactionDataList; } + /** + * Warning: this method will walk through the entire data directory + * Do not call it too frequently as it could create high disk load + * in environments with a large amount of hosted data. + * @return a list of paths that are being hosted + */ + public List findAllHostedPaths() { + Path dataPath = Paths.get(Settings.getInstance().getDataPath()); + Path tempPath = Paths.get(Settings.getInstance().getTempDataPath()); + + // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length + // Also exclude the _temp and _misc paths if present + List allPaths = new ArrayList<>(); + try { + allPaths = Files.walk(dataPath, 3) + .filter(Files::isDirectory) + .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString()) + && !path.toString().contains("_misc") + && path.getFileName().toString().length() > 32) + .collect(Collectors.toList()); + } + catch (IOException e) { + LOGGER.info("Unable to walk through hosted data: {}", e.getMessage()); + } + + return allPaths; + } + public void invalidateHostedTransactionsCache() { this.hostedTransactions = null; }