Use the /.qortal/patch file to apply differences when merging together two layers, rather than walking through the file tree like we did before.

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -200,8 +200,11 @@ public class ArbitraryDataDiff {
     }
 
     private void writeMetadata() throws IOException {
-        ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths,
+        ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.diffPath);
-                this.removedPaths, this.diffPath, this.previousSignature);
+        metadata.setAddedPaths(this.addedPaths);
+        metadata.setModifiedPaths(this.modifiedPaths);
+        metadata.setRemovedPaths(this.removedPaths);
+        metadata.setPreviousSignature(this.previousSignature);
         metadata.write();
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java

@@ -3,15 +3,13 @@ package org.qortal.arbitrary;
 import org.apache.commons.io.FileUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.qortal.crypto.Crypto;
 import org.qortal.settings.Settings;
 import org.qortal.utils.FilesystemUtils;
 
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.*;
-import java.nio.file.attribute.BasicFileAttributes;
+import java.util.List;
-import java.util.Arrays;
 import java.util.UUID;
 
 public class ArbitraryDataMerge {
@@ -22,6 +20,7 @@ public class ArbitraryDataMerge {
     private Path pathAfter;
     private Path mergePath;
     private String identifier;
+    private ArbitraryDataMetadata metadata;
 
     public ArbitraryDataMerge(Path pathBefore, Path pathAfter) {
         this.pathBefore = pathBefore;
@@ -32,7 +31,8 @@ public class ArbitraryDataMerge {
         try {
             this.preExecute();
             this.copyPreviousStateToMergePath();
-            this.findDifferences();
+            this.loadMetadata();
+            this.applyDifferences();
 
         } finally {
             this.postExecute();
@@ -68,97 +68,35 @@ public class ArbitraryDataMerge {
         ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get(""));
     }
 
-    private void findDifferences() {
+    private void loadMetadata() throws IOException {
-        final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
+        this.metadata = new ArbitraryDataMetadata(this.pathAfter);
-        final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
+        this.metadata.read();
-        final Path mergePathAbsolute = this.mergePath.toAbsolutePath();
+    }
 
-//        LOGGER.info("this.pathBefore: {}", this.pathBefore);
+    private void applyDifferences() throws IOException {
-//        LOGGER.info("this.pathAfter: {}", this.pathAfter);
-//        LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
-//        LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
-//        LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute);
 
+        List<Path> addedPaths = this.metadata.getAddedPaths();
+        for (Path path : addedPaths) {
+            LOGGER.info("File was added: {}", path.toString());
+            Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
+            ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path);
+        }
 
-        try {
+        List<Path> modifiedPaths = this.metadata.getModifiedPaths();
-            // Check for additions or modifications
+        for (Path path : modifiedPaths) {
-            Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
+            LOGGER.info("File was modified: {}", path.toString());
+            Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
+            ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path);
+        }
 
-                @Override
+        List<Path> removedPaths = this.metadata.getRemovedPaths();
-                public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
+        for (Path path : removedPaths) {
-                    return FileVisitResult.CONTINUE;
+            LOGGER.info("File was removed: {}", path.toString());
-                }
+            ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path);
-
-                @Override
-                public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException {
-                    Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath());
-                    Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter);
-
-                    boolean wasAdded = false;
-                    boolean wasModified = false;
-                    boolean wasRemoved = false;
-
-                    if (after.toString().endsWith(".removed")) {
-                        LOGGER.trace("File was removed: {}", after.toString());
-                        wasRemoved = true;
-                    }
-                    else if (!Files.exists(filePathBefore)) {
-                        LOGGER.trace("File was added: {}", after.toString());
-                        wasAdded = true;
-                    }
-                    else if (Files.size(after) != Files.size(filePathBefore)) {
-                        // Check file size first because it's quicker
-                        LOGGER.trace("File size was modified: {}", after.toString());
-                        wasModified = true;
-                    }
-                    else if (!Arrays.equals(ArbitraryDataMerge.digestFromPath(after), ArbitraryDataMerge.digestFromPath(filePathBefore))) {
-                        // Check hashes as a last resort
-                        LOGGER.trace("File contents were modified: {}", after.toString());
-                        wasModified = true;
-                    }
-
-                    if (wasAdded | wasModified) {
-                        ArbitraryDataMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter);
-                    }
-
-                    if (wasRemoved) {
-                        if (filePathAfter.toString().endsWith(".removed")) {
-                            // Trim the ".removed"
-                            Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8));
-                            ArbitraryDataMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed);
-                        }
-                    }
-
-                    return FileVisitResult.CONTINUE;
-                }
-
-                @Override
-                public FileVisitResult visitFileFailed(Path file, IOException e){
-                    LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
-                    // TODO: throw exception?
-                    return FileVisitResult.TERMINATE;
-                }
-
-                @Override
-                public FileVisitResult postVisitDirectory(Path dir, IOException e) {
-                    return FileVisitResult.CONTINUE;
-                }
-
-            });
-        } catch (IOException e) {
-            LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
         }
     }
 
 
-    private static byte[] digestFromPath(Path path) {
-        try {
-            return Crypto.digest(Files.readAllBytes(path));
-        } catch (IOException e) {
-            return null;
-        }
-    }
-
     private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
         if (!Files.exists(source)) {
             throw new IOException(String.format("File not found: %s", source.toString()));

src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java

@@ -7,11 +7,13 @@ import org.json.JSONObject;
 import org.qortal.utils.Base58;
 
 import java.io.BufferedWriter;
+import java.io.File;
 import java.io.FileWriter;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.ArrayList;
 import java.util.List;
 
 public class ArbitraryDataMetadata {
@@ -27,13 +29,18 @@ public class ArbitraryDataMetadata {
 
     private String jsonString;
 
-    public ArbitraryDataMetadata(List<Path> addedPaths, List<Path> modifiedPaths, List<Path> removedPaths,
+    public ArbitraryDataMetadata(Path filePath) {
-                                 Path filePath, byte[] previousSignature) {
-        this.addedPaths = addedPaths;
-        this.modifiedPaths = modifiedPaths;
-        this.removedPaths = removedPaths;
         this.filePath = filePath;
-        this.previousSignature = previousSignature;
+        this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal");
+
+        this.addedPaths = new ArrayList<>();
+        this.modifiedPaths = new ArrayList<>();
+        this.removedPaths = new ArrayList<>();
+    }
+
+    public void read() throws IOException {
+        this.loadJson();
+        this.readJson();
     }
 
     public void write() throws IOException {
@@ -42,39 +49,123 @@ public class ArbitraryDataMetadata {
         this.writeToQortalPath();
     }
 
+
+    private void loadJson() throws IOException {
+        Path path = Paths.get(this.qortalDirectoryPath.toString(), "patch");
+        File patchFile = new File(path.toString());
+        if (!patchFile.exists()) {
+            throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
+        }
+
+        this.jsonString = new String(Files.readAllBytes(path));
+    }
+
+    private void readJson() {
+        if (this.jsonString == null) {
+            throw new IllegalStateException("Patch JSON string is null");
+        }
+
+        JSONObject patch = new JSONObject(this.jsonString);
+        if (patch.has("prevSig")) {
+            String prevSig = (String)patch.get("prevSig");
+            if (prevSig != null) {
+                this.previousSignature = Base58.decode(prevSig);
+            }
+        }
+        if (patch.has("added")) {
+            JSONArray added = (JSONArray) patch.get("added");
+            if (added != null) {
+                for (int i=0; i<added.length(); i++) {
+                    String pathString = (String)added.get(i);
+                    this.addedPaths.add(Paths.get(pathString));
+                }
+            }
+        }
+        if (patch.has("modified")) {
+            JSONArray modified = (JSONArray) patch.get("modified");
+            if (modified != null) {
+                for (int i=0; i<modified.length(); i++) {
+                    String pathString = (String)modified.get(i);
+                    this.modifiedPaths.add(Paths.get(pathString));
+                }
+            }
+        }
+        if (patch.has("removed")) {
+            JSONArray removed = (JSONArray) patch.get("removed");
+            if (removed != null) {
+                for (int i=0; i<removed.length(); i++) {
+                    String pathString = (String)removed.get(i);
+                    this.removedPaths.add(Paths.get(pathString));
+                }
+            }
+        }
+    }
+
+
+
     private void buildJson() {
         JSONArray addedPathsJson = new JSONArray(this.addedPaths);
         JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths);
         JSONArray removedPathsJson = new JSONArray(this.removedPaths);
         String previousSignature58 = Base58.encode(this.previousSignature);
 
-        JSONObject jsonObject = new JSONObject();
+        JSONObject patch = new JSONObject();
-        jsonObject.put("prevSig", previousSignature58);
+        patch.put("prevSig", previousSignature58);
-        jsonObject.put("added", addedPathsJson);
+        patch.put("added", addedPathsJson);
-        jsonObject.put("modified", modifiedPathsJson);
+        patch.put("modified", modifiedPathsJson);
-        jsonObject.put("removed", removedPathsJson);
+        patch.put("removed", removedPathsJson);
 
-        this.jsonString = jsonObject.toString(4);
+        this.jsonString = patch.toString(2);
     }
 
     private void createQortalDirectory() {
-        Path qortalDir = Paths.get(this.filePath.toString(), ".qortal");
         try {
-            Files.createDirectories(qortalDir);
+            Files.createDirectories(this.qortalDirectoryPath);
         } catch (IOException e) {
             throw new IllegalStateException("Unable to create .qortal directory");
         }
-        this.qortalDirectoryPath = qortalDir;
     }
 
     private void writeToQortalPath() throws IOException {
-        Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
+        Path patchPath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
-        BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString()));
+        BufferedWriter writer = new BufferedWriter(new FileWriter(patchPath.toString()));
         writer.write(this.jsonString);
         writer.close();
     }
 
 
+    public void setAddedPaths(List<Path> addedPaths) {
+        this.addedPaths = addedPaths;
+    }
+
+    public List<Path> getAddedPaths() {
+        return this.addedPaths;
+    }
+
+    public void setModifiedPaths(List<Path> modifiedPaths) {
+        this.modifiedPaths = modifiedPaths;
+    }
+
+    public List<Path> getModifiedPaths() {
+        return this.modifiedPaths;
+    }
+
+    public void setRemovedPaths(List<Path> removedPaths) {
+        this.removedPaths = removedPaths;
+    }
+
+    public List<Path> getRemovedPaths() {
+        return this.removedPaths;
+    }
+
+    public void setPreviousSignature(byte[] previousSignature) {
+        this.previousSignature = previousSignature;
+    }
+
+    public byte[] getPreviousSignature() {
+        return this.getPreviousSignature();
+    }
+
     public String getJsonString() {
         return this.jsonString;
     }

src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java

@@ -133,11 +133,28 @@ public class ArbitraryDataWriter {
         }
 
         // Validate the patch
+        this.validatePatch();
+    }
+
+    private void validatePatch() throws IOException {
         if (this.filePath == null) {
             throw new IllegalStateException("Null path after creating patch");
         }
-        if (FilesystemUtils.isDirectoryEmpty(this.filePath)) {
+
-            throw new IllegalStateException("Patch has no content. Either no files have changed, or something went wrong");
+        File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile();
+        if (!qortalMetadataDirectoryFile.exists()) {
+            throw new IllegalStateException("Qortal metadata folder doesn't exist in patch");
+        }
+        if (!qortalMetadataDirectoryFile.isDirectory()) {
+            throw new IllegalStateException("Qortal metadata folder isn't a directory");
+        }
+
+        File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile();
+        if (!qortalPatchMetadataFile.exists()) {
+            throw new IllegalStateException("Qortal patch metadata file doesn't exist in patch");
+        }
+        if (!qortalPatchMetadataFile.isFile()) {
+            throw new IllegalStateException("Qortal patch metadata file isn't a file");
         }
     }