From 16ac92b2efc5c0e6eeb92a255dd6bab1fddedf05 Mon Sep 17 00:00:00 2001 From: CalDescent Date: Sun, 15 Aug 2021 10:13:32 +0100 Subject: [PATCH] Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals. --- .../arbitrary/ArbitraryDataBuilder.java | 20 +++++ .../arbitrary/ArbitraryDataCreatePatch.java | 23 +++++- .../qortal/arbitrary/ArbitraryDataDiff.java | 61 +++++++++----- .../arbitrary/ArbitraryDataMetadata.java | 82 +++++++++++++++++++ .../qortal/arbitrary/ArbitraryDataWriter.java | 5 +- .../org/qortal/utils/FilesystemUtils.java | 21 ++++- 6 files changed, 185 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java index d90ef43c..7bbf962d 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java @@ -28,6 +28,7 @@ public class ArbitraryDataBuilder { private List transactions; private ArbitraryTransactionData latestPutTransaction; private List paths; + private byte[] latestSignature; private Path finalPath; public ArbitraryDataBuilder(String name, Service service) { @@ -41,6 +42,7 @@ public class ArbitraryDataBuilder { this.validateTransactions(); this.processTransactions(); this.validatePaths(); + this.findLatestSignature(); this.buildLatestState(); } @@ -119,6 +121,20 @@ public class ArbitraryDataBuilder { } } + private void findLatestSignature() { + if (this.transactions.size() == 0) { + throw new IllegalStateException("Unable to find latest signature from empty transaction list"); + } + + // Find the latest signature + ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1); + if (latestTransaction == null) { + throw new IllegalStateException("Unable to find latest signature from null transaction"); + } + + this.latestSignature = latestTransaction.getSignature(); + } + private void validatePaths() { if (this.paths == null || this.paths.isEmpty()) { throw new IllegalStateException(String.format("No paths available from which to build latest state")); @@ -149,4 +165,8 @@ public class ArbitraryDataBuilder { return this.finalPath; } + public byte[] getLatestSignature() { + return this.latestSignature; + } + } diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java index ed4095a4..6d28c455 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java @@ -3,6 +3,7 @@ package org.qortal.arbitrary; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.qortal.repository.DataException; +import org.qortal.utils.FilesystemUtils; import java.io.IOException; import java.nio.file.Files; @@ -14,11 +15,13 @@ public class ArbitraryDataCreatePatch { private Path pathBefore; private Path pathAfter; + private byte[] previousSignature; private Path finalPath; - public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter) { + public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; + this.previousSignature = previousSignature; } public void create() throws DataException, IOException { @@ -26,6 +29,10 @@ public class ArbitraryDataCreatePatch { this.preExecute(); this.process(); + } catch (Exception e) { + this.cleanupOnFailure(); + throw e; + } finally { this.postExecute(); } @@ -44,11 +51,19 @@ public class ArbitraryDataCreatePatch { } - private void process() { + private void cleanupOnFailure() { + try { + FilesystemUtils.safeDeleteDirectory(this.finalPath, true); + } catch (IOException e) { + LOGGER.info("Unable to cleanup diff directory on failure"); + } + } - ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter); - diff.compute(); + private void process() throws IOException { + + ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature); this.finalPath = diff.getDiffPath(); + diff.compute(); } public Path getFinalPath() { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java index 1952ddb0..b59db487 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java @@ -9,7 +9,9 @@ import java.io.File; import java.io.IOException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.UUID; public class ArbitraryDataDiff { @@ -18,19 +20,33 @@ public class ArbitraryDataDiff { private Path pathBefore; private Path pathAfter; + private byte[] previousSignature; private Path diffPath; private String identifier; - public ArbitraryDataDiff(Path pathBefore, Path pathAfter) { + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + + public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) { this.pathBefore = pathBefore; this.pathAfter = pathAfter; + this.previousSignature = previousSignature; + + this.addedPaths = new ArrayList<>(); + this.modifiedPaths = new ArrayList<>(); + this.removedPaths = new ArrayList<>(); + + this.createRandomIdentifier(); + this.createOutputDirectory(); } - public void compute() { + public void compute() throws IOException { try { this.preExecute(); this.findAddedOrModifiedFiles(); this.findRemovedFiles(); + this.writeMetadata(); } finally { this.postExecute(); @@ -38,8 +54,7 @@ public class ArbitraryDataDiff { } private void preExecute() { - this.createRandomIdentifier(); - this.createOutputDirectory(); + } private void postExecute() { @@ -63,18 +78,12 @@ public class ArbitraryDataDiff { } private void findAddedOrModifiedFiles() { - final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); - final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); - -// LOGGER.info("this.pathBefore: {}", this.pathBefore); -// LOGGER.info("this.pathAfter: {}", this.pathAfter); -// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute); -// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute); -// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute); - - try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + // Check for additions or modifications Files.walkFileTree(this.pathAfter, new FileVisitor() { @@ -93,16 +102,19 @@ public class ArbitraryDataDiff { if (!Files.exists(filePathBefore)) { LOGGER.info("File was added: {}", after.toString()); + diff.addedPaths.add(filePathAfter); wasAdded = true; } else if (Files.size(after) != Files.size(filePathBefore)) { // Check file size first because it's quicker LOGGER.info("File size was modified: {}", after.toString()); + diff.modifiedPaths.add(filePathAfter); wasModified = true; } else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) { // Check hashes as a last resort LOGGER.info("File contents were modified: {}", after.toString()); + diff.modifiedPaths.add(filePathAfter); wasModified = true; } @@ -133,10 +145,12 @@ public class ArbitraryDataDiff { } private void findRemovedFiles() { - final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); - final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); - final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); try { + final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath(); + final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath(); + final Path diffPathAbsolute = this.diffPath.toAbsolutePath(); + final ArbitraryDataDiff diff = this; + // Check for removals Files.walkFileTree(this.pathBefore, new FileVisitor() { @@ -147,10 +161,9 @@ public class ArbitraryDataDiff { if (!Files.exists(directoryPathAfter)) { LOGGER.info("Directory was removed: {}", directoryPathAfter.toString()); - + diff.removedPaths.add(directoryPathBefore); ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore); // TODO: we might need to mark directories differently to files - // TODO: add path to manifest JSON } return FileVisitResult.CONTINUE; @@ -163,9 +176,9 @@ public class ArbitraryDataDiff { if (!Files.exists(filePathAfter)) { LOGGER.trace("File was removed: {}", before.toString()); + diff.removedPaths.add(filePathBefore); ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore); - // TODO: add path to manifest JSON } return FileVisitResult.CONTINUE; @@ -189,6 +202,12 @@ public class ArbitraryDataDiff { } } + private void writeMetadata() throws IOException { + ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths, + this.removedPaths, this.diffPath, this.previousSignature); + metadata.write(); + } + private static byte[] digestFromPath(Path path) { try { diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java new file mode 100644 index 00000000..7b5f7c22 --- /dev/null +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java @@ -0,0 +1,82 @@ +package org.qortal.arbitrary; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.json.JSONArray; +import org.json.JSONObject; +import org.qortal.utils.Base58; + +import java.io.BufferedWriter; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +public class ArbitraryDataMetadata { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class); + + private List addedPaths; + private List modifiedPaths; + private List removedPaths; + private Path filePath; + private Path qortalDirectoryPath; + private byte[] previousSignature; + + private String jsonString; + + public ArbitraryDataMetadata(List addedPaths, List modifiedPaths, List removedPaths, + Path filePath, byte[] previousSignature) { + this.addedPaths = addedPaths; + this.modifiedPaths = modifiedPaths; + this.removedPaths = removedPaths; + this.filePath = filePath; + this.previousSignature = previousSignature; + } + + public void write() throws IOException { + this.buildJson(); + this.createQortalDirectory(); + this.writeToQortalPath(); + } + + private void buildJson() { + JSONArray addedPathsJson = new JSONArray(this.addedPaths); + JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths); + JSONArray removedPathsJson = new JSONArray(this.removedPaths); + String previousSignature58 = Base58.encode(this.previousSignature); + + JSONObject jsonObject = new JSONObject(); + jsonObject.put("prevSig", previousSignature58); + jsonObject.put("added", addedPathsJson); + jsonObject.put("modified", modifiedPathsJson); + jsonObject.put("removed", removedPathsJson); + + this.jsonString = jsonObject.toString(4); + } + + private void createQortalDirectory() { + Path qortalDir = Paths.get(this.filePath.toString(), ".qortal"); + try { + Files.createDirectories(qortalDir); + } catch (IOException e) { + throw new IllegalStateException("Unable to create .qortal directory"); + } + this.qortalDirectoryPath = qortalDir; + } + + private void writeToQortalPath() throws IOException { + Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch"); + BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString())); + writer.write(this.jsonString); + writer.close(); + } + + + public String getJsonString() { + return this.jsonString; + } + +} diff --git a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java index 4fe21bb6..61b5c8eb 100644 --- a/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java +++ b/src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java @@ -117,9 +117,12 @@ public class ArbitraryDataWriter { builder.build(); Path builtPath = builder.getFinalPath(); + // Obtain the latest signature, so this can be included in the patch + byte[] latestSignature = builder.getLatestSignature(); + // Compute a diff of the latest changes on top of the previous state // Then use only the differences as our data payload - ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath); + ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature); patch.create(); this.filePath = patch.getFinalPath(); diff --git a/src/main/java/org/qortal/utils/FilesystemUtils.java b/src/main/java/org/qortal/utils/FilesystemUtils.java index b3f32c91..1bfdea78 100644 --- a/src/main/java/org/qortal/utils/FilesystemUtils.java +++ b/src/main/java/org/qortal/utils/FilesystemUtils.java @@ -80,7 +80,6 @@ public class FilesystemUtils { // Delete existing if (FilesystemUtils.pathInsideDataOrTempPath(source)) { - System.out.println(String.format("Deleting file %s", source.toString())); Files.delete(source); } @@ -142,6 +141,26 @@ public class FilesystemUtils { } } + public static void safeDeleteDirectory(Path path, boolean cleanup) throws IOException { + // Delete path, if it exists in our data/temp directory + if (FilesystemUtils.pathInsideDataOrTempPath(path)) { + File directory = new File(path.toString()); + FileUtils.deleteDirectory(directory); + } + + if (cleanup) { + // Delete the parent directory if it is empty (and exists in our data/temp directory) + Path parentDirectory = path.getParent(); + if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) { + try { + Files.deleteIfExists(parentDirectory); + } catch (IOException e) { + // This part is optional, so ignore failures + } + } + } + } + public static boolean pathInsideDataOrTempPath(Path path) { if (path == null) { return false;