Take a hash of the previous state's directory structure and file contents, and then include that hash in the patch metadata (when creating a new patch). This allows the integrity of the layers to be validated as each one is applied.

src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java

@@ -22,6 +22,7 @@ public class ArbitraryDataDiff {
     private Path pathBefore;
     private Path pathAfter;
     private byte[] previousSignature;
+    private byte[] previousHash;
     private Path diffPath;
     private String identifier;
 
@@ -45,6 +46,7 @@ public class ArbitraryDataDiff {
     public void compute() throws IOException {
         try {
             this.preExecute();
+            this.hashPreviousState();
             this.findAddedOrModifiedFiles();
             this.findRemovedFiles();
             this.writeMetadata();
@@ -78,6 +80,12 @@ public class ArbitraryDataDiff {
         this.diffPath = tempDir;
     }
 
+    private void hashPreviousState() throws IOException {
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore);
+        digest.compute();
+        this.previousHash = digest.getHash();
+    }
+
     private void findAddedOrModifiedFiles() {
         try {
             final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
@@ -220,6 +228,7 @@ public class ArbitraryDataDiff {
         metadata.setModifiedPaths(this.modifiedPaths);
         metadata.setRemovedPaths(this.removedPaths);
         metadata.setPreviousSignature(this.previousSignature);
+        metadata.setPreviousHash(this.previousHash);
         metadata.write();
     }
 

src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java

@@ -0,0 +1,77 @@
+package org.qortal.arbitrary;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.qortal.utils.Base58;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class ArbitraryDataDigest {
+
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDigest.class);
+
+    private Path path;
+    private byte[] hash;
+
+    public ArbitraryDataDigest(Path path) {
+        this.path = path;
+    }
+
+    public void compute() throws IOException {
+        List<Path> allPaths = new ArrayList<>();
+        Files.walk(path).filter(Files::isRegularFile).forEachOrdered(p -> allPaths.add(p));
+        Path basePathAbsolute = this.path.toAbsolutePath();
+
+        MessageDigest sha256 = null;
+        try {
+            sha256 = MessageDigest.getInstance("SHA-256");
+        } catch (NoSuchAlgorithmException e) {
+            throw new IllegalStateException("SHA-256 hashing algorithm unavailable");
+        }
+
+        for (Path path : allPaths) {
+            // We need to work with paths relative to the base path, to ensure the same hash
+            // is generated on different systems
+            Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath());
+
+            // Exclude Qortal folder since it can be different each time
+            // We only care about hashing the actual user data
+            if (relativePath.startsWith(".qortal/")) {
+                continue;
+            }
+
+            // Hash path
+            byte[] filePathBytes = relativePath.toString().toLowerCase().getBytes(StandardCharsets.UTF_8);
+            sha256.update(filePathBytes);
+
+            // Hash contents
+            byte[] fileContent = Files.readAllBytes(path);
+            sha256.update(fileContent);
+        }
+        this.hash = sha256.digest();
+    }
+
+    public boolean isHashValid(byte[] hash) {
+        return Arrays.equals(hash, this.hash);
+    }
+
+    public byte[] getHash() {
+        return this.hash;
+    }
+
+    public String getHash58() {
+        if (this.hash == null) {
+            return null;
+        }
+        return Base58.encode(this.hash);
+    }
+
+}

src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java

@@ -1,20 +1,27 @@
 package org.qortal.arbitrary.metadata;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.json.JSONArray;
 import org.json.JSONObject;
 import org.qortal.utils.Base58;
 
+import java.lang.reflect.Field;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.ArrayList;
+import java.util.LinkedHashMap;
 import java.util.List;
 
 public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
 
+    private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class);
+
     private List<Path> addedPaths;
     private List<Path> modifiedPaths;
     private List<Path> removedPaths;
     private byte[] previousSignature;
+    private byte[] previousHash;
 
     public ArbitraryDataMetadataPatch(Path filePath) {
         super(filePath);
@@ -42,6 +49,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
                 this.previousSignature = Base58.decode(prevSig);
             }
         }
+        if (patch.has("prevHash")) {
+            String prevHash = patch.getString("prevHash");
+            if (prevHash != null) {
+                this.previousHash = Base58.decode(prevHash);
+            }
+        }
         if (patch.has("added")) {
             JSONArray added = (JSONArray) patch.get("added");
             if (added != null) {
@@ -74,7 +87,17 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
     @Override
     protected void buildJson() {
         JSONObject patch = new JSONObject();
+        // Attempt to use a LinkedHashMap so that the order of fields is maintained
+        try {
+            Field changeMap = patch.getClass().getDeclaredField("map");
+            changeMap.setAccessible(true);
+            changeMap.set(patch, new LinkedHashMap<>());
+            changeMap.setAccessible(false);
+        } catch (IllegalAccessException | NoSuchFieldException e) {
+            // Don't worry about failures as this is for ordering only
+        }
         patch.put("prevSig", Base58.encode(this.previousSignature));
+        patch.put("prevHash", Base58.encode(this.previousHash));
         patch.put("added", new JSONArray(this.addedPaths));
         patch.put("modified", new JSONArray(this.modifiedPaths));
         patch.put("removed", new JSONArray(this.removedPaths));
@@ -116,4 +139,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
         return this.previousSignature;
     }
 
+    public void setPreviousHash(byte[] previousHash) {
+        this.previousHash = previousHash;
+    }
+
+    public byte[] getPreviousHash() {
+        return this.previousHash;
+    }
+
 }

src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java

@@ -0,0 +1,58 @@
+package org.qortal.test.arbitrary;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.qortal.arbitrary.ArbitraryDataDigest;
+import org.qortal.repository.DataException;
+import org.qortal.test.common.Common;
+
+import java.io.FileWriter;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.UUID;
+
+import static org.junit.Assert.*;
+
+public class ArbitraryDataDigestTests extends Common {
+
+    @Before
+    public void beforeTest() throws DataException {
+        Common.useDefaultSettings();
+    }
+
+    @Test
+    public void testDirectoryDigest() throws IOException {
+        Path dataPath = Paths.get("src/test/resources/arbitrary/demo1");
+        String expectedHash58 = "59dw8CgVybcHAUL5GYgYUUfFffVVhiMKZLCnULPKT6oC";
+
+        // Ensure directory exists
+        assertTrue(dataPath.toFile().exists());
+        assertTrue(dataPath.toFile().isDirectory());
+
+        // Compute a hash
+        ArbitraryDataDigest digest = new ArbitraryDataDigest(dataPath);
+        digest.compute();
+        assertEquals(expectedHash58, digest.getHash58());
+
+        // Write a random file to .qortal/cache to ensure it isn't being included in the digest function
+        // We exclude all .qortal files from the digest since they can be different with each build, and
+        // we only care about the actual user files
+        FileWriter fileWriter = new FileWriter(Paths.get(dataPath.toString(), ".qortal", "cache").toString());
+        fileWriter.append(UUID.randomUUID().toString());
+        fileWriter.close();
+
+        // Recompute the hash
+        digest = new ArbitraryDataDigest(dataPath);
+        digest.compute();
+        assertEquals(expectedHash58, digest.getHash58());
+
+        // Now compute the hash 100 more times to ensure it's always the same
+        for (int i=0; i<100; i++) {
+            digest = new ArbitraryDataDigest(dataPath);
+            digest.compute();
+            assertEquals(expectedHash58, digest.getHash58());
+        }
+    }
+
+}

src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java

@@ -1,4 +1,4 @@
-package org.qortal.test;
+package org.qortal.test.arbitrary;
 
 import org.junit.Before;
 import org.junit.Test;
@@ -10,7 +10,7 @@ import java.util.Random;
 
 import static org.junit.Assert.*;
 
-public class DataTests extends Common {
+public class ArbitraryDataFileTests extends Common {
 
 	@Before
 	public void beforeTest() throws DataException {

src/test/resources/arbitrary/demo1/.qortal/cache

@@ -0,0 +1 @@
+2ea5a99a-3b85-4f1f-a259-436787f90bd1

src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt

@@ -0,0 +1 @@
+Pellentesque laoreet laoreet dui ut volutpat.

src/test/resources/arbitrary/demo1/dir1/lorem4.txt

@@ -0,0 +1 @@
+Sed non lacus ante.

src/test/resources/arbitrary/demo1/lorem1.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.

src/test/resources/arbitrary/demo1/lorem2.txt

@@ -0,0 +1 @@
+Quisque viverra neque quis eros dapibus

src/test/resources/arbitrary/demo1/lorem3.txt

@@ -0,0 +1 @@
+Sed ac magna pretium, suscipit mauris sed, ultrices nunc.

src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt

@@ -0,0 +1 @@
+Pellentesque laoreet laoreet dui ut volutpat.

src/test/resources/arbitrary/demo2/dir1/lorem4.txt

@@ -0,0 +1 @@
+Sed non lacus ante.

src/test/resources/arbitrary/demo2/lorem1.txt

@@ -0,0 +1 @@
+Lorem ipsum dolor sit amet, consectetur adipiscing elit.

src/test/resources/arbitrary/demo2/lorem2.txt

@@ -0,0 +1 @@
+Quisque viverra neque quis eros dapibus

src/test/resources/arbitrary/demo2/lorem3.txt

@@ -0,0 +1 @@
+Sed ac magna pretium, suscipit mauris sed, ultrices nunc.