AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Write patch metadata to a file inside a hidden ".qortal" folder which is included with each patch. This can be used in place of the existing ".removed" placeholder files to track removals.

Browse Source
This commit is contained in:
CalDescent 2021-08-15 10:13:32 +01:00
parent f095964f7b
commit 16ac92b2ef
6 changed files with 185 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -28,6 +28,7 @@ public class ArbitraryDataBuilder {
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
private List<Path> paths;
private byte[] latestSignature;
private Path finalPath;
public ArbitraryDataBuilder(String name, Service service) {
@ -41,6 +42,7 @@ public class ArbitraryDataBuilder {
this.validateTransactions();
this.processTransactions();
this.validatePaths();
this.findLatestSignature();
this.buildLatestState();
}
@ -119,6 +121,20 @@ public class ArbitraryDataBuilder {
}
}
private void findLatestSignature() {
if (this.transactions.size() == 0) {
throw new IllegalStateException("Unable to find latest signature from empty transaction list");
}
// Find the latest signature
ArbitraryTransactionData latestTransaction = this.transactions.get(this.transactions.size() - 1);
if (latestTransaction == null) {
throw new IllegalStateException("Unable to find latest signature from null transaction");
}
this.latestSignature = latestTransaction.getSignature();
}
private void validatePaths() {
if (this.paths == null || this.paths.isEmpty()) {
throw new IllegalStateException(String.format("No paths available from which to build latest state"));
@ -149,4 +165,8 @@ public class ArbitraryDataBuilder {
return this.finalPath;
}
public byte[] getLatestSignature() {
return this.latestSignature;
}
}

23
src/main/java/org/qortal/arbitrary/ArbitraryDataCreatePatch.java
View File

@ -3,6 +3,7 @@ package org.qortal.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.repository.DataException;
import org.qortal.utils.FilesystemUtils;
import java.io.IOException;
import java.nio.file.Files;
@ -14,11 +15,13 @@ public class ArbitraryDataCreatePatch {
private Path pathBefore;
private Path pathAfter;
private byte[] previousSignature;
private Path finalPath;
public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter) {
public ArbitraryDataCreatePatch(Path pathBefore, Path pathAfter, byte[] previousSignature) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
this.previousSignature = previousSignature;
}
public void create() throws DataException, IOException {
@ -26,6 +29,10 @@ public class ArbitraryDataCreatePatch {
this.preExecute();
this.process();
} catch (Exception e) {
this.cleanupOnFailure();
throw e;
} finally {
this.postExecute();
}
@ -44,11 +51,19 @@ public class ArbitraryDataCreatePatch {
}
private void process() {
private void cleanupOnFailure() {
try {
FilesystemUtils.safeDeleteDirectory(this.finalPath, true);
} catch (IOException e) {
LOGGER.info("Unable to cleanup diff directory on failure");
}
}
ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter);
diff.compute();
private void process() throws IOException {
ArbitraryDataDiff diff = new ArbitraryDataDiff(this.pathBefore, this.pathAfter, this.previousSignature);
this.finalPath = diff.getDiffPath();
diff.compute();
}
public Path getFinalPath() {

61
src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java
View File

@ -9,7 +9,9 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
public class ArbitraryDataDiff {
@ -18,19 +20,33 @@ public class ArbitraryDataDiff {
private Path pathBefore;
private Path pathAfter;
private byte[] previousSignature;
private Path diffPath;
private String identifier;
public ArbitraryDataDiff(Path pathBefore, Path pathAfter) {
private List<Path> addedPaths;
private List<Path> modifiedPaths;
private List<Path> removedPaths;
public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
this.previousSignature = previousSignature;
this.addedPaths = new ArrayList<>();
this.modifiedPaths = new ArrayList<>();
this.removedPaths = new ArrayList<>();
this.createRandomIdentifier();
this.createOutputDirectory();
}
public void compute() {
public void compute() throws IOException {
try {
this.preExecute();
this.findAddedOrModifiedFiles();
this.findRemovedFiles();
this.writeMetadata();
} finally {
this.postExecute();
@ -38,8 +54,7 @@ public class ArbitraryDataDiff {
}
private void preExecute() {
this.createRandomIdentifier();
this.createOutputDirectory();
}
private void postExecute() {
@ -63,18 +78,12 @@ public class ArbitraryDataDiff {
}
private void findAddedOrModifiedFiles() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
// LOGGER.info("this.pathBefore: {}", this.pathBefore);
// LOGGER.info("this.pathAfter: {}", this.pathAfter);
// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute);
try {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
final ArbitraryDataDiff diff = this;
// Check for additions or modifications
Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
@ -93,16 +102,19 @@ public class ArbitraryDataDiff {
if (!Files.exists(filePathBefore)) {
LOGGER.info("File was added: {}", after.toString());
diff.addedPaths.add(filePathAfter);
wasAdded = true;
}
else if (Files.size(after) != Files.size(filePathBefore)) {
// Check file size first because it's quicker
LOGGER.info("File size was modified: {}", after.toString());
diff.modifiedPaths.add(filePathAfter);
wasModified = true;
}
else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) {
// Check hashes as a last resort
LOGGER.info("File contents were modified: {}", after.toString());
diff.modifiedPaths.add(filePathAfter);
wasModified = true;
}
@ -133,10 +145,12 @@ public class ArbitraryDataDiff {
}
private void findRemovedFiles() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
try {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
final ArbitraryDataDiff diff = this;
// Check for removals
Files.walkFileTree(this.pathBefore, new FileVisitor<Path>() {
@ -147,10 +161,9 @@ public class ArbitraryDataDiff {
if (!Files.exists(directoryPathAfter)) {
LOGGER.info("Directory was removed: {}", directoryPathAfter.toString());
diff.removedPaths.add(directoryPathBefore);
ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore);
// TODO: we might need to mark directories differently to files
// TODO: add path to manifest JSON
}
return FileVisitResult.CONTINUE;
@ -163,9 +176,9 @@ public class ArbitraryDataDiff {
if (!Files.exists(filePathAfter)) {
LOGGER.trace("File was removed: {}", before.toString());
diff.removedPaths.add(filePathBefore);
ArbitraryDataDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore);
// TODO: add path to manifest JSON
}
return FileVisitResult.CONTINUE;
@ -189,6 +202,12 @@ public class ArbitraryDataDiff {
}
}
private void writeMetadata() throws IOException {
ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths,
this.removedPaths, this.diffPath, this.previousSignature);
metadata.write();
}
private static byte[] digestFromPath(Path path) {
try {

82
src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java Normal file
View File

@ -0,0 +1,82 @@
package org.qortal.arbitrary;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.qortal.utils.Base58;
import java.io.BufferedWriter;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
public class ArbitraryDataMetadata {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadata.class);
private List<Path> addedPaths;
private List<Path> modifiedPaths;
private List<Path> removedPaths;
private Path filePath;
private Path qortalDirectoryPath;
private byte[] previousSignature;
private String jsonString;
public ArbitraryDataMetadata(List<Path> addedPaths, List<Path> modifiedPaths, List<Path> removedPaths,
Path filePath, byte[] previousSignature) {
this.addedPaths = addedPaths;
this.modifiedPaths = modifiedPaths;
this.removedPaths = removedPaths;
this.filePath = filePath;
this.previousSignature = previousSignature;
}
public void write() throws IOException {
this.buildJson();
this.createQortalDirectory();
this.writeToQortalPath();
}
private void buildJson() {
JSONArray addedPathsJson = new JSONArray(this.addedPaths);
JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths);
JSONArray removedPathsJson = new JSONArray(this.removedPaths);
String previousSignature58 = Base58.encode(this.previousSignature);
JSONObject jsonObject = new JSONObject();
jsonObject.put("prevSig", previousSignature58);
jsonObject.put("added", addedPathsJson);
jsonObject.put("modified", modifiedPathsJson);
jsonObject.put("removed", removedPathsJson);
this.jsonString = jsonObject.toString(4);
}
private void createQortalDirectory() {
Path qortalDir = Paths.get(this.filePath.toString(), ".qortal");
try {
Files.createDirectories(qortalDir);
} catch (IOException e) {
throw new IllegalStateException("Unable to create .qortal directory");
}
this.qortalDirectoryPath = qortalDir;
}
private void writeToQortalPath() throws IOException {
Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString()));
writer.write(this.jsonString);
writer.close();
}
public String getJsonString() {
return this.jsonString;
}
}

5
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -117,9 +117,12 @@ public class ArbitraryDataWriter {
builder.build();
Path builtPath = builder.getFinalPath();
// Obtain the latest signature, so this can be included in the patch
byte[] latestSignature = builder.getLatestSignature();
// Compute a diff of the latest changes on top of the previous state
// Then use only the differences as our data payload
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath);
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(builtPath, this.filePath, latestSignature);
patch.create();
this.filePath = patch.getFinalPath();

21
src/main/java/org/qortal/utils/FilesystemUtils.java
View File

@ -80,7 +80,6 @@ public class FilesystemUtils {
// Delete existing
if (FilesystemUtils.pathInsideDataOrTempPath(source)) {
System.out.println(String.format("Deleting file %s", source.toString()));
Files.delete(source);
}
@ -142,6 +141,26 @@ public class FilesystemUtils {
}
}
public static void safeDeleteDirectory(Path path, boolean cleanup) throws IOException {
// Delete path, if it exists in our data/temp directory
if (FilesystemUtils.pathInsideDataOrTempPath(path)) {
File directory = new File(path.toString());
FileUtils.deleteDirectory(directory);
}
if (cleanup) {
// Delete the parent directory if it is empty (and exists in our data/temp directory)
Path parentDirectory = path.getParent();
if (FilesystemUtils.pathInsideDataOrTempPath(parentDirectory)) {
try {
Files.deleteIfExists(parentDirectory);
} catch (IOException e) {
// This part is optional, so ignore failures
}
}
}
}
public static boolean pathInsideDataOrTempPath(Path path) {
if (path == null) {
return false;