From 0f1927b4b1dd1a82afd54f7fd5cd533dabb1b210 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Tue, 17 Aug 2021 08:43:08 +0100 Subject: [PATCH] Take a hash of the previous state's directory structure and file contents, and then include that hash in the patch metadata (when creating a new patch). This allows the integrity of the layers to be validated as each one is applied. --- .../qortal/arbitrary/ArbitraryDataDiff.java | 9 +++ .../qortal/arbitrary/ArbitraryDataDigest.java | 77 +++++++++++++++++++ .../metadata/ArbitraryDataMetadataPatch.java | 31 ++++++++ .../arbitrary/ArbitraryDataDigestTests.java | 58 ++++++++++++++ .../ArbitraryDataFileTests.java} | 4 +- .../resources/arbitrary/demo1/.qortal/cache | 1 + .../arbitrary/demo1/dir1/dir2/lorem5.txt | 1 + .../resources/arbitrary/demo1/dir1/lorem4.txt | 1 + src/test/resources/arbitrary/demo1/lorem1.txt | 1 + src/test/resources/arbitrary/demo1/lorem2.txt | 1 + src/test/resources/arbitrary/demo1/lorem3.txt | 1 + .../arbitrary/demo2/dir1/dir2/lorem5.txt | 1 + .../resources/arbitrary/demo2/dir1/lorem4.txt | 1 + src/test/resources/arbitrary/demo2/lorem1.txt | 1 + src/test/resources/arbitrary/demo2/lorem2.txt | 1 + src/test/resources/arbitrary/demo2/lorem3.txt | 1 + 16 files changed, 188 insertions(+), 2 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java create mode 100644 src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java rename src/test/java/org/qortal/test/{DataTests.java => arbitrary/ArbitraryDataFileTests.java} (96%) create mode 100644 src/test/resources/arbitrary/demo1/.qortal/cache create mode 100644 src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt create mode 100644 src/test/resources/arbitrary/demo1/dir1/lorem4.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem1.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem2.txt create mode 100644 src/test/resources/arbitrary/demo1/lorem3.txt create mode 100644 src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt create mode 100644 src/test/resources/arbitrary/demo2/dir1/lorem4.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem1.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem2.txt create mode 100644 src/test/resources/arbitrary/demo2/lorem3.txt diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 8db1af09..c8f2c2b6 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -22,6 +22,7 @@ public class ArbitraryDataDiff { private Path pathBefore; private Path pathAfter; private byte[] previousSignature; + private byte[] previousHash; private Path diffPath; private String identifier; @@ -45,6 +46,7 @@ public class ArbitraryDataDiff { public void compute() throws IOException { try { this.preExecute(); + this.hashPreviousState(); this.findAddedOrModifiedFiles(); this.findRemovedFiles(); this.writeMetadata(); @@ -78,6 +80,12 @@ public class ArbitraryDataDiff { this.diffPath = tempDir; } + private void hashPreviousState() throws IOException { + ArbitraryDataDigest digest = new ArbitraryDataDigest(this.pathBefore); + digest.compute(); + this.previousHash = digest.getHash(); + } + private void findAddedOrModifiedFiles() { try { final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); @@ -220,6 +228,7 @@ public class ArbitraryDataDiff { metadata.setModifiedPaths(this.modifiedPaths); metadata.setRemovedPaths(this.removedPaths); metadata.setPreviousSignature(this.previousSignature); + metadata.setPreviousHash(this.previousHash); metadata.write(); } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java new file mode 100644 index 00000000..315d79a7 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDigest.java @@ -0,0 +1,77 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.utils.Base58; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ArbitraryDataDigest { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDigest.class); + + private Path path; + private byte[] hash; + + public ArbitraryDataDigest(Path path) { + this.path = path; + } + + public void compute() throws IOException { + List allPaths = new ArrayList<>(); + Files.walk(path).filter(Files::isRegularFile).forEachOrdered(p -> allPaths.add(p)); + Path basePathAbsolute = this.path.toAbsolutePath(); + + MessageDigest sha256 = null; + try { + sha256 = MessageDigest.getInstance("SHA-256"); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 hashing algorithm unavailable"); + } + + for (Path path : allPaths) { + // We need to work with paths relative to the base path, to ensure the same hash + // is generated on different systems + Path relativePath = basePathAbsolute.relativize(path.toAbsolutePath()); + + // Exclude Qortal folder since it can be different each time + // We only care about hashing the actual user data + if (relativePath.startsWith(".qortal/")) { + continue; + } + + // Hash path + byte[] filePathBytes = relativePath.toString().toLowerCase().getBytes(StandardCharsets.UTF_8); + sha256.update(filePathBytes); + + // Hash contents + byte[] fileContent = Files.readAllBytes(path); + sha256.update(fileContent); + } + this.hash = sha256.digest(); + } + + public boolean isHashValid(byte[] hash) { + return Arrays.equals(hash, this.hash); + } + + public byte[] getHash() { + return this.hash; + } + + public String getHash58() { + if (this.hash == null) { + return null; + } + return Base58.encode(this.hash); + } + +} diff --git a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java index fd68bec3..e6f392cb 100644 --- a/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java +++ b/src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java @@ -1,20 +1,27 @@ package org.qortal.arbitrary.metadata; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.json.JSONArray; import org.json.JSONObject; import org.qortal.utils.Base58; +import java.lang.reflect.Field; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class); + private List addedPaths; private List modifiedPaths; private List removedPaths; private byte[] previousSignature; + private byte[] previousHash; public ArbitraryDataMetadataPatch(Path filePath) { super(filePath); @@ -42,6 +49,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { this.previousSignature = Base58.decode(prevSig); } } + if (patch.has("prevHash")) { + String prevHash = patch.getString("prevHash"); + if (prevHash != null) { + this.previousHash = Base58.decode(prevHash); + } + } if (patch.has("added")) { JSONArray added = (JSONArray) patch.get("added"); if (added != null) { @@ -74,7 +87,17 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { @Override protected void buildJson() { JSONObject patch = new JSONObject(); + // Attempt to use a LinkedHashMap so that the order of fields is maintained + try { + Field changeMap = patch.getClass().getDeclaredField("map"); + changeMap.setAccessible(true); + changeMap.set(patch, new LinkedHashMap<>()); + changeMap.setAccessible(false); + } catch (IllegalAccessException | NoSuchFieldException e) { + // Don't worry about failures as this is for ordering only + } patch.put("prevSig", Base58.encode(this.previousSignature)); + patch.put("prevHash", Base58.encode(this.previousHash)); patch.put("added", new JSONArray(this.addedPaths)); patch.put("modified", new JSONArray(this.modifiedPaths)); patch.put("removed", new JSONArray(this.removedPaths)); @@ -116,4 +139,12 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata { return this.previousSignature; } + public void setPreviousHash(byte[] previousHash) { + this.previousHash = previousHash; + } + + public byte[] getPreviousHash() { + return this.previousHash; + } + } diff --git a/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java new file mode 100644 index 00000000..b2772a28 --- /dev/null +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataDigestTests.java @@ -0,0 +1,58 @@ +package org.qortal.test.arbitrary; + +import org.junit.Before; +import org.junit.Test; +import org.qortal.arbitrary.ArbitraryDataDigest; +import org.qortal.repository.DataException; +import org.qortal.test.common.Common; + +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.UUID; + +import static org.junit.Assert.*; + +public class ArbitraryDataDigestTests extends Common { + + @Before + public void beforeTest() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testDirectoryDigest() throws IOException { + Path dataPath = Paths.get("src/test/resources/arbitrary/demo1"); + String expectedHash58 = "59dw8CgVybcHAUL5GYgYUUfFffVVhiMKZLCnULPKT6oC"; + + // Ensure directory exists + assertTrue(dataPath.toFile().exists()); + assertTrue(dataPath.toFile().isDirectory()); + + // Compute a hash + ArbitraryDataDigest digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Write a random file to .qortal/cache to ensure it isn't being included in the digest function + // We exclude all .qortal files from the digest since they can be different with each build, and + // we only care about the actual user files + FileWriter fileWriter = new FileWriter(Paths.get(dataPath.toString(), ".qortal", "cache").toString()); + fileWriter.append(UUID.randomUUID().toString()); + fileWriter.close(); + + // Recompute the hash + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + + // Now compute the hash 100 more times to ensure it's always the same + for (int i=0; i<100; i++) { + digest = new ArbitraryDataDigest(dataPath); + digest.compute(); + assertEquals(expectedHash58, digest.getHash58()); + } + } + +} diff --git a/src/test/java/org/qortal/test/DataTests.java b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java similarity index 96% rename from src/test/java/org/qortal/test/DataTests.java rename to src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java index 1d515239..bb016920 100644 --- a/src/test/java/org/qortal/test/DataTests.java +++ b/src/test/java/org/qortal/test/arbitrary/ArbitraryDataFileTests.java @@ -1,4 +1,4 @@ -package org.qortal.test; +package org.qortal.test.arbitrary; import org.junit.Before; import org.junit.Test; @@ -10,7 +10,7 @@ import java.util.Random; import static org.junit.Assert.*; -public class DataTests extends Common { +public class ArbitraryDataFileTests extends Common { @Before public void beforeTest() throws DataException { diff --git a/src/test/resources/arbitrary/demo1/.qortal/cache b/src/test/resources/arbitrary/demo1/.qortal/cache new file mode 100644 index 00000000..d10d5a03 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/.qortal/cache @@ -0,0 +1 @@ +2ea5a99a-3b85-4f1f-a259-436787f90bd1 \ No newline at end of file diff --git a/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt new file mode 100644 index 00000000..ef07da1f --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. diff --git a/src/test/resources/arbitrary/demo1/dir1/lorem4.txt b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt new file mode 100644 index 00000000..80d1dda7 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/dir1/lorem4.txt @@ -0,0 +1 @@ +Sed non lacus ante. diff --git a/src/test/resources/arbitrary/demo1/lorem1.txt b/src/test/resources/arbitrary/demo1/lorem1.txt new file mode 100644 index 00000000..4f006a88 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem1.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. diff --git a/src/test/resources/arbitrary/demo1/lorem2.txt b/src/test/resources/arbitrary/demo1/lorem2.txt new file mode 100644 index 00000000..8a9c4367 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem2.txt @@ -0,0 +1 @@ +Quisque viverra neque quis eros dapibus diff --git a/src/test/resources/arbitrary/demo1/lorem3.txt b/src/test/resources/arbitrary/demo1/lorem3.txt new file mode 100644 index 00000000..5db7e985 --- /dev/null +++ b/src/test/resources/arbitrary/demo1/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc. diff --git a/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt new file mode 100644 index 00000000..ef07da1f --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/dir2/lorem5.txt @@ -0,0 +1 @@ +Pellentesque laoreet laoreet dui ut volutpat. diff --git a/src/test/resources/arbitrary/demo2/dir1/lorem4.txt b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt new file mode 100644 index 00000000..80d1dda7 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/dir1/lorem4.txt @@ -0,0 +1 @@ +Sed non lacus ante. diff --git a/src/test/resources/arbitrary/demo2/lorem1.txt b/src/test/resources/arbitrary/demo2/lorem1.txt new file mode 100644 index 00000000..4f006a88 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem1.txt @@ -0,0 +1 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. diff --git a/src/test/resources/arbitrary/demo2/lorem2.txt b/src/test/resources/arbitrary/demo2/lorem2.txt new file mode 100644 index 00000000..8a9c4367 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem2.txt @@ -0,0 +1 @@ +Quisque viverra neque quis eros dapibus diff --git a/src/test/resources/arbitrary/demo2/lorem3.txt b/src/test/resources/arbitrary/demo2/lorem3.txt new file mode 100644 index 00000000..5db7e985 --- /dev/null +++ b/src/test/resources/arbitrary/demo2/lorem3.txt @@ -0,0 +1 @@ +Sed ac magna pretium, suscipit mauris sed, ultrices nunc.