Delete directories in the data folder that have no associated transaction.

This adds support for "unconfirmable" data uploads, which will be useful for Q-Chat. It also handles cases where a transaction is orphaned and then subsequently becomes invalid.

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataCleanupManager.java

@@ -204,6 +204,9 @@ public class ArbitraryDataCleanupManager extends Thread {
 
 				try (final Repository repository = RepositoryManager.getRepository()) {
 
+					// Check if there are any hosted files that don't have matching transactions
+					this.checkForExpiredTransactions(repository);
+
 					// Delete additional data at random if we're over our storage limit
 					// Use a threshold of 1, for the same reasons as above
 					if (!storageManager.isStorageSpaceAvailable(1.0f)) {
@@ -233,6 +236,45 @@ public class ArbitraryDataCleanupManager extends Thread {
 		}
 	}
 
+	public List<Path> findPathsWithNoAssociatedTransaction(Repository repository) {
+		List<Path> pathList = new ArrayList<>();
+
+		// Find all hosted paths
+		List<Path> allPaths = ArbitraryDataStorageManager.getInstance().findAllHostedPaths();
+
+		// Loop through each path and find those without matching signatures
+		for (Path path : allPaths) {
+			try {
+				String[] contents = path.toFile().list();
+				if (contents == null || contents.length == 0) {
+					// Ignore empty directories
+					continue;
+				}
+
+				String signature58 = path.getFileName().toString();
+				byte[] signature = Base58.decode(signature58);
+				TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
+				if (transactionData == null) {
+					// No transaction data, and no DataException, so we can assume that this data relates to an expired transaction
+					pathList.add(path);
+				}
+
+			} catch (DataException e) {
+				continue;
+			}
+		}
+
+		return pathList;
+	}
+
+	private void checkForExpiredTransactions(Repository repository) {
+		List<Path> expiredPaths = this.findPathsWithNoAssociatedTransaction(repository);
+		for (Path expiredPath : expiredPaths) {
+			LOGGER.info("Found path with no associated transaction: {}", expiredPath.toString());
+			this.safeDeleteDirectory(expiredPath.toFile(), "no matching transaction");
+		}
+	}
+
 	private void storageLimitReached(Repository repository) throws InterruptedException {
 		// We think that the storage limit has been reached
 

src/main/java/org/qortal/controller/arbitrary/ArbitraryDataStorageManager.java

@@ -244,28 +244,14 @@ public class ArbitraryDataStorageManager extends Thread {
 
         List<ArbitraryTransactionData> arbitraryTransactionDataList = new ArrayList<>();
 
-        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        // Find all hosted paths
-        Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
+        List<Path> allPaths = this.findAllHostedPaths();
-
-        // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
-        // Also exclude the _temp and _misc paths if present
-        List<Path> allPaths = new ArrayList<>();
-        try {
-            allPaths = Files.walk(dataPath, 3)
-                    .filter(Files::isDirectory)
-                    .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
-                            && !path.toString().contains("_misc")
-                            && path.getFileName().toString().length() > 32)
-                    .collect(Collectors.toList());
-        }
-        catch (IOException e) {
-            LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
-        }
 
         // Loop through each path and attempt to match it to a signature
         for (Path path : allPaths) {
             try {
-                if (path.toFile().list().length == 0) {
+                String[] contents = path.toFile().list();
+                if (contents == null || contents.length == 0) {
                     // Ignore empty directories
                     continue;
                 }
@@ -289,6 +275,34 @@ public class ArbitraryDataStorageManager extends Thread {
         return arbitraryTransactionDataList;
     }
 
+    /**
+     * Warning: this method will walk through the entire data directory
+     * Do not call it too frequently as it could create high disk load
+     * in environments with a large amount of hosted data.
+     * @return a list of paths that are being hosted
+     */
+    public List<Path> findAllHostedPaths() {
+        Path dataPath = Paths.get(Settings.getInstance().getDataPath());
+        Path tempPath = Paths.get(Settings.getInstance().getTempDataPath());
+
+        // Walk through 3 levels of the file tree and find directories that are greater than 32 characters in length
+        // Also exclude the _temp and _misc paths if present
+        List<Path> allPaths = new ArrayList<>();
+        try {
+            allPaths = Files.walk(dataPath, 3)
+                    .filter(Files::isDirectory)
+                    .filter(path -> !path.toAbsolutePath().toString().contains(tempPath.toAbsolutePath().toString())
+                            && !path.toString().contains("_misc")
+                            && path.getFileName().toString().length() > 32)
+                    .collect(Collectors.toList());
+        }
+        catch (IOException e) {
+            LOGGER.info("Unable to walk through hosted data: {}", e.getMessage());
+        }
+
+        return allPaths;
+    }
+
     public void invalidateHostedTransactionsCache() {
         this.hostedTransactions = null;
     }