Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals.

src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java

@@ -28,6 +28,7 @@ public class ArbitraryDataBuilder {
     private List<ArbitraryTransactionData> transactions;
     private ArbitraryTransactionData latestPutTransaction;
     private List<Path> paths;
+    private byte[] latestSignature;
     private Path finalPath;
 
     public ArbitraryDataBuilder(String name, Service service) {
@@ -41,6 +42,7 @@ public class ArbitraryDataBuilder {
         this.validateTransactions();
         this.processTransactions();
         this.validatePaths();
+        this.findLatestSignature();
         this.buildLatestState();
     }
 
@@ -119,6 +121,20 @@ public class ArbitraryDataBuilder {
         }
     }
 
+    private void findLatestSignature() {
+        if (this.transactions.size() == 0) {
+            throw new IllegalStateException("Unable to find latest signature from empty transaction list");
+        }
+
+        // Find the latest signature
+        ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1);
+        if (latestTransaction == null) {
+            throw new IllegalStateException("Unable to find latest signature from null transaction");
+        }
+
+        this.latestSignature = latestTransaction.getSignature();
+    }
+
     private void validatePaths() {
         if (this.paths == null || this.paths.isEmpty()) {
             throw new IllegalStateException(String.format("No paths available from which to build latest state"));
@@ -149,4 +165,8 @@ public class ArbitraryDataBuilder {
         return this.finalPath;
     }
 
+    public byte[] getLatestSignature() {
+        return this.latestSignature;
+    }
+
 }

src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java

@@ -3,6 +3,7 @@ package org.qortal.arbitrary;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.qortal.repository.DataException;
+import org.qortal.utils.FilesystemUtils;
 
 import java.io.IOException;
 import java.nio.file.Files;
@@ -14,11 +15,13 @@ public class ArbitraryDataCreatePatch {
 
     private Path pathBefore;
     private Path pathAfter;
+    private byte[] previousSignature;
     private Path finalPath;
 
-    public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter) {
+    public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) {
         this.pathBefore = pathBefore;
         this.pathAfter = pathAfter;
+        this.previousSignature = previousSignature;
     }
 
     public void create() throws DataException, IOException {
@@ -26,6 +29,10 @@ public class ArbitraryDataCreatePatch {
             this.preExecute();
             this.process();
 
+        } catch (Exception e) {
+            this.cleanupOnFailure();
+            throw e;
+
         } finally {
             this.postExecute();
         }
@@ -44,11 +51,19 @@ public class ArbitraryDataCreatePatch {
 
     }
 
-    private void process() {
+    private void cleanupOnFailure() {
+        try {
+            FilesystemUtils.safeDeleteDirectory(this.finalPath, true);
+        } catch (IOException e) {
+            LOGGER.info("Unable to cleanup diff directory on failure");
+        }
+    }
 
-        ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter);
+    private void process() throws IOException {
-        diff.compute();
+
+        ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature);
         this.finalPath = diff.getDiffPath();
+        diff.compute();
     }
 
     public Path getFinalPath() {

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -9,7 +9,9 @@ import java.io.File;
 import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 import java.util.UUID;
 
 public class ArbitraryDataDiff {
@@ -18,19 +20,33 @@ public class ArbitraryDataDiff {
 
     private Path pathBefore;
     private Path pathAfter;
+    private byte[] previousSignature;
     private Path diffPath;
     private String identifier;
 
-    public ArbitraryDataDiff(Path pathBefore, Path pathAfter) {
+    private List<Path> addedPaths;
+    private List<Path> modifiedPaths;
+    private List<Path> removedPaths;
+
+    public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) {
         this.pathBefore = pathBefore;
         this.pathAfter = pathAfter;
+        this.previousSignature = previousSignature;
+
+        this.addedPaths = new ArrayList<>();
+        this.modifiedPaths = new ArrayList<>();
+        this.removedPaths = new ArrayList<>();
+
+        this.createRandomIdentifier();
+        this.createOutputDirectory();
     }
 
-    public void compute() {
+    public void compute() throws IOException {
         try {
             this.preExecute();
             this.findAddedOrModifiedFiles();
             this.findRemovedFiles();
+            this.writeMetadata();
 
         } finally {
             this.postExecute();
@@ -38,8 +54,7 @@ public class ArbitraryDataDiff {
     }
 
     private void preExecute() {
-        this.createRandomIdentifier();
+
-        this.createOutputDirectory();
     }
 
     private void postExecute() {
@@ -63,18 +78,12 @@ public class ArbitraryDataDiff {
     }
 
     private void findAddedOrModifiedFiles() {
-        final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
-        final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
-        final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
-
-//        LOGGER.info("this.pathBefore: {}", this.pathBefore);
-//        LOGGER.info("this.pathAfter: {}", this.pathAfter);
-//        LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
-//        LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
-//        LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute);
-
-
         try {
+            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
+            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
+            final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
+            final ArbitraryDataDiff diff = this;
+
             // Check for additions or modifications
             Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
 
@@ -93,16 +102,19 @@ public class ArbitraryDataDiff {
 
                     if (!Files.exists(filePathBefore)) {
                         LOGGER.info("File was added: {}", after.toString());
+                        diff.addedPaths.add(filePathAfter);
                         wasAdded = true;
                     }
                     else if (Files.size(after) != Files.size(filePathBefore)) {
                         // Check file size first because it's quicker
                         LOGGER.info("File size was modified: {}", after.toString());
+                        diff.modifiedPaths.add(filePathAfter);
                         wasModified = true;
                     }
                     else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) {
                         // Check hashes as a last resort
                         LOGGER.info("File contents were modified: {}", after.toString());
+                        diff.modifiedPaths.add(filePathAfter);
                         wasModified = true;
                     }
 
@@ -133,10 +145,12 @@ public class ArbitraryDataDiff {
     }
 
     private void findRemovedFiles() {
-        final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
-        final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
-        final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
         try {
+            final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
+            final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
+            final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
+            final ArbitraryDataDiff diff = this;
+
             // Check for removals
             Files.walkFileTree(this.pathBefore, new FileVisitor<Path>() {
 
@@ -147,10 +161,9 @@ public class ArbitraryDataDiff {
 
                     if (!Files.exists(directoryPathAfter)) {
                         LOGGER.info("Directory was removed: {}", directoryPathAfter.toString());
-
+                        diff.removedPaths.add(directoryPathBefore);
                         ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore);
                         // TODO: we might need to mark directories differently to files
-                        // TODO: add path to manifest JSON
                     }
 
                     return FileVisitResult.CONTINUE;
@@ -163,9 +176,9 @@ public class ArbitraryDataDiff {
 
                     if (!Files.exists(filePathAfter)) {
                         LOGGER.trace("File was removed: {}", before.toString());
+                        diff.removedPaths.add(filePathBefore);
 
                         ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore);
-                        // TODO: add path to manifest JSON
                     }
 
                     return FileVisitResult.CONTINUE;
@@ -189,6 +202,12 @@ public class ArbitraryDataDiff {
         }
     }
 
+    private void writeMetadata() throws IOException {
+        ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths,
+                this.removedPaths, this.diffPath, this.previousSignature);
+        metadata.write();
+    }
+
 
     private static byte[] digestFromPath(Path path) {
         try {

src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java

@@ -0,0 +1,82 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.json.JSONArray;
+import org.json.JSONObject;
+import org.qortal.utils.Base58;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+
+public class ArbitraryDataMetadata {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class);
+
+    private List<Path> addedPaths;
+    private List<Path> modifiedPaths;
+    private List<Path> removedPaths;
+    private Path filePath;
+    private Path qortalDirectoryPath;
+    private byte[] previousSignature;
+
+    private String jsonString;
+
+    public ArbitraryDataMetadata(List<Path> addedPaths, List<Path> modifiedPaths, List<Path> removedPaths,
+                                 Path filePath, byte[] previousSignature) {
+        this.addedPaths = addedPaths;
+        this.modifiedPaths = modifiedPaths;
+        this.removedPaths = removedPaths;
+        this.filePath = filePath;
+        this.previousSignature = previousSignature;
+    }
+
+    public void write() throws IOException {
+        this.buildJson();
+        this.createQortalDirectory();
+        this.writeToQortalPath();
+    }
+
+    private void buildJson() {
+        JSONArray addedPathsJson = new JSONArray(this.addedPaths);
+        JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths);
+        JSONArray removedPathsJson = new JSONArray(this.removedPaths);
+        String previousSignature58 = Base58.encode(this.previousSignature);
+
+        JSONObject jsonObject = new JSONObject();
+        jsonObject.put("prevSig", previousSignature58);
+        jsonObject.put("added", addedPathsJson);
+        jsonObject.put("modified", modifiedPathsJson);
+        jsonObject.put("removed", removedPathsJson);
+
+        this.jsonString = jsonObject.toString(4);
+    }
+
+    private void createQortalDirectory() {
+        Path qortalDir = Paths.get(this.filePath.toString(), ".qortal");
+        try {
+            Files.createDirectories(qortalDir);
+        } catch (IOException e) {
+            throw new IllegalStateException("Unable to create .qortal directory");
+        }
+        this.qortalDirectoryPath = qortalDir;
+    }
+
+    private void writeToQortalPath() throws IOException {
+        Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
+        BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString()));
+        writer.write(this.jsonString);
+        writer.close();
+    }
+
+
+    public String getJsonString() {
+        return this.jsonString;
+    }
+
+}

src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java

@@ -117,9 +117,12 @@ public class ArbitraryDataWriter {
         builder.build();
         Path builtPath = builder.getFinalPath();
 
+        // Obtain the latest signature, so this can be included in the patch
+        byte[] latestSignature = builder.getLatestSignature();
+
         // Compute a diff of the latest changes on top of the previous state
         // Then use only the differences as our data payload
-        ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath);
+        ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature);
         patch.create();
         this.filePath = patch.getFinalPath();
 

src/main/java/org/qortal/utils/FilesystemUtils.java

@@ -80,7 +80,6 @@ public class FilesystemUtils {
 
         // Delete existing
         if (FilesystemUtils.pathInsideDataOrTempPath(source)) {
-            System.out.println(String.format("Deleting file %s", source.toString()));
             Files.delete(source);
         }
 
@@ -142,6 +141,26 @@ public class FilesystemUtils {
         }
     }
 
+    public static void safeDeleteDirectory(Path path, boolean cleanup) throws IOException {
+        // Delete path, if it exists in our data/temp directory
+        if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
+            File directory = new File(path.toString());
+            FileUtils.deleteDirectory(directory);
+        }
+
+        if (cleanup) {
+            // Delete the parent directory if it is empty (and exists in our data/temp directory)
+            Path parentDirectory = path.getParent();
+            if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
+                try {
+                    Files.deleteIfExists(parentDirectory);
+                } catch (IOException e) {
+                    // This part is optional, so ignore failures
+                }
+            }
+        }
+    }
+
     public static boolean pathInsideDataOrTempPath(Path path) {
         if (path == null) {
             return false;