crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Significant rework of patches

Browse Source 
- The "diff type" is now specified per file, allowing for different diff methods in each modified file.
- Patches will only be created when both the before and after files are less than 100kiB in size.
- Patches are validated after creation, and if invalid it will fall back to including the entire file.

This has identified a bug where patching fails for files without trailing newline characters, which still needs to be fixed. Until then, it will fall back to including the entire file in these cases.
This commit is contained in:
CalDescent 2021-10-24 10:47:47 +01:00
parent 12b3267d5c
commit 8dd4d71d75
9 changed files with 634 additions and 174 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -29,3 +29,4 @@
/*.7z
/tmp
/data*
/src/test/resources/arbitrary/*/.qortal/cache

2
src/main/java/org/qortal/arbitrary/ArbitraryDataBuilder.java
View File

@ -145,7 +145,7 @@ public class ArbitraryDataBuilder {
}
}
private void buildLatestState() throws IOException {
private void buildLatestState() throws IOException, DataException {
if (this.paths.size() == 1) {
// No patching needed
this.finalPath = this.paths.get(0);

8
src/main/java/org/qortal/arbitrary/ArbitraryDataCombiner.java
View File

@ -4,6 +4,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.repository.DataException;
import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils;
@ -31,7 +32,7 @@ public class ArbitraryDataCombiner {
this.signatureBefore = signatureBefore;
}
public void combine() throws IOException {
public void combine() throws IOException, DataException {
try {
this.preExecute();
this.readMetadata();
@ -125,9 +126,8 @@ public class ArbitraryDataCombiner {
}
}
private void process() throws IOException {
String patchType = metadata.getPatchType();
ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter, patchType);
private void process() throws IOException, DataException {
ArbitraryDataMerge merge = new ArbitraryDataMerge(this.pathBefore, this.pathAfter);
merge.compute();
this.finalPath = merge.getMergePath();
}

183
src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java
View File

@ -1,26 +1,64 @@
package org.qortal.arbitrary;
import com.github.difflib.DiffUtils;
import com.github.difflib.UnifiedDiffUtils;
import com.github.difflib.patch.Patch;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONObject;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.arbitrary.patch.UnifiedDiffPatch;
import org.qortal.crypto.Crypto;
import org.qortal.settings.Settings;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
import java.util.*;
public class ArbitraryDataDiff {
/** Only create a patch if both the before and after file sizes are within defined limit **/
private static long MAX_DIFF_FILE_SIZE = 100 * 1024L; // 100kiB
public enum DiffType {
COMPLETE_FILE,
UNIFIED_DIFF
}
public static class ModifiedPath {
private Path path;
private DiffType diffType;
public ModifiedPath(Path path, DiffType diffType) {
this.path = path;
this.diffType = diffType;
}
public ModifiedPath(JSONObject jsonObject) {
String pathString = jsonObject.getString("path");
if (pathString != null) {
this.path = Paths.get(pathString);
}
String diffTypeString = jsonObject.getString("type");
if (diffTypeString != null) {
this.diffType = DiffType.valueOf(diffTypeString);
}
}
public Path getPath() {
return this.path;
}
public DiffType getDiffType() {
return this.diffType;
}
public String toString() {
return this.path.toString();
}
}
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataDiff.class);
private Path pathBefore;
@ -31,7 +69,7 @@ public class ArbitraryDataDiff {
private String identifier;
private List<Path> addedPaths;
private List<Path> modifiedPaths;
private List<ModifiedPath> modifiedPaths;
private List<Path> removedPaths;
public ArbitraryDataDiff(Path pathBefore, Path pathAfter, byte[] previousSignature) {
@ -91,7 +129,7 @@ public class ArbitraryDataDiff {
this.previousHash = digest.getHash();
}
private void findAddedOrModifiedFiles() {
private void findAddedOrModifiedFiles() throws IOException {
try {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
@ -107,11 +145,11 @@ public class ArbitraryDataDiff {
}
@Override
public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException {
Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath());
Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter);
public FileVisitResult visitFile(Path afterPathAbsolute, BasicFileAttributes attrs) throws IOException {
Path afterPathRelative = pathAfterAbsolute.relativize(afterPathAbsolute.toAbsolutePath());
Path beforePathAbsolute = pathBeforeAbsolute.resolve(afterPathRelative);
if (filePathAfter.startsWith(".qortal")) {
if (afterPathRelative.startsWith(".qortal")) {
// Ignore the .qortal metadata folder
return FileVisitResult.CONTINUE;
}
@ -119,31 +157,27 @@ public class ArbitraryDataDiff {
boolean wasAdded = false;
boolean wasModified = false;
if (!Files.exists(filePathBefore)) {
LOGGER.info("File was added: {}", filePathAfter.toString());
diff.addedPaths.add(filePathAfter);
if (!Files.exists(beforePathAbsolute)) {
LOGGER.info("File was added: {}", afterPathRelative.toString());
diff.addedPaths.add(afterPathRelative);
wasAdded = true;
}
else if (Files.size(after) != Files.size(filePathBefore)) {
else if (Files.size(afterPathAbsolute) != Files.size(beforePathAbsolute)) {
// Check file size first because it's quicker
LOGGER.info("File size was modified: {}", filePathAfter.toString());
diff.modifiedPaths.add(filePathAfter);
LOGGER.info("File size was modified: {}", afterPathRelative.toString());
wasModified = true;
}
else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(after), ArbitraryDataDiff.digestFromPath(filePathBefore))) {
else if (!Arrays.equals(ArbitraryDataDiff.digestFromPath(afterPathAbsolute), ArbitraryDataDiff.digestFromPath(beforePathAbsolute))) {
// Check hashes as a last resort
LOGGER.info("File contents were modified: {}", filePathAfter.toString());
diff.modifiedPaths.add(filePathAfter);
LOGGER.info("File contents were modified: {}", afterPathRelative.toString());
wasModified = true;
}
if (wasAdded) {
ArbitraryDataDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter);
diff.copyFilePathToBaseDir(afterPathAbsolute, diffPathAbsolute, afterPathRelative);
}
if (wasModified) {
// Create patch using java-diff-utils
Path destination = Paths.get(diffPathAbsolute.toString(), filePathAfter.toString());
ArbitraryDataDiff.createAndCopyDiffUtilsPatch(filePathBefore, after, destination);
diff.pathModified(beforePathAbsolute, afterPathAbsolute, afterPathRelative, diffPathAbsolute);
}
return FileVisitResult.CONTINUE;
@ -163,8 +197,8 @@ public class ArbitraryDataDiff {
});
} catch (IOException e) {
// TODO: throw exception?
LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
throw(e);
}
}
@ -240,7 +274,6 @@ public class ArbitraryDataDiff {
private void writeMetadata() throws IOException {
ArbitraryDataMetadataPatch metadata = new ArbitraryDataMetadataPatch(this.diffPath);
metadata.setPatchType("unified-diff");
metadata.setAddedPaths(this.addedPaths);
metadata.setModifiedPaths(this.modifiedPaths);
metadata.setRemovedPaths(this.removedPaths);
@ -250,15 +283,38 @@ public class ArbitraryDataDiff {
}
private static byte[] digestFromPath(Path path) {
try {
return Crypto.digest(Files.readAllBytes(path));
} catch (IOException e) {
return null;
private void pathModified(Path beforePathAbsolute, Path afterPathAbsolute, Path afterPathRelative,
Path destinationBasePathAbsolute) throws IOException {
Path destination = Paths.get(destinationBasePathAbsolute.toString(), afterPathRelative.toString());
long beforeSize = Files.size(beforePathAbsolute);
long afterSize = Files.size(afterPathAbsolute);
DiffType diffType;
if (beforeSize > MAX_DIFF_FILE_SIZE || afterSize > MAX_DIFF_FILE_SIZE) {
// Files are large, so don't attempt a diff
this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
diffType = DiffType.COMPLETE_FILE;
}
else {
// Attempt to create patch using java-diff-utils
UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(beforePathAbsolute, afterPathAbsolute, destination);
unifiedDiffPatch.create();
if (unifiedDiffPatch.isValid()) {
diffType = DiffType.UNIFIED_DIFF;
}
else {
// Diff failed validation, so copy the whole file instead
this.copyFilePathToBaseDir(afterPathAbsolute, destinationBasePathAbsolute, afterPathRelative);
diffType = DiffType.COMPLETE_FILE;
}
}
ModifiedPath modifiedPath = new ModifiedPath(afterPathRelative, diffType);
this.modifiedPaths.add(modifiedPath);
}
private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
private void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
if (!Files.exists(source)) {
throw new IOException(String.format("File not found: %s", source.toString()));
}
@ -274,54 +330,21 @@ public class ArbitraryDataDiff {
LOGGER.trace("Copying {} to {}", source, dest);
Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
}
private static void createAndCopyDiffUtilsPatch(Path before, Path after, Path destination) throws IOException {
if (!Files.exists(before)) {
throw new IOException(String.format("File not found (before): %s", before.toString()));
}
if (!Files.exists(after)) {
throw new IOException(String.format("File not found (after): %s", after.toString()));
}
// Ensure parent folders exist in the destination
File file = new File(destination.toString());
File parent = file.getParentFile();
if (parent != null) {
parent.mkdirs();
}
// Delete an existing file if it exists
File destFile = destination.toFile();
if (destFile.exists() && destFile.isFile()) {
Files.delete(destination);
}
// Load the two files into memory
List<String> original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8);
List<String> revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8);
// Generate diff information
Patch<String> diff = DiffUtils.diff(original, revised);
// Generate unified diff format
String originalFileName = before.getFileName().toString();
String revisedFileName = after.getFileName().toString();
List<String> unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0);
// Write the diff to the destination directory
FileWriter fileWriter = new FileWriter(destination.toString(), true);
BufferedWriter writer = new BufferedWriter(fileWriter);
for (String line : unifiedDiff) {
writer.append(line);
writer.newLine();
}
writer.flush();
writer.close();
}
public Path getDiffPath() {
return this.diffPath;
}
// Utils
private static byte[] digestFromPath(Path path) {
try {
return Crypto.digest(Files.readAllBytes(path));
} catch (IOException e) {
return null;
}
}
}

83
src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java
View File

@ -1,24 +1,19 @@
package org.qortal.arbitrary;
import com.github.difflib.DiffUtils;
import com.github.difflib.UnifiedDiffUtils;
import com.github.difflib.patch.Patch;
import com.github.difflib.patch.PatchFailedException;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.arbitrary.ArbitraryDataDiff.*;
import org.qortal.arbitrary.metadata.ArbitraryDataMetadataPatch;
import org.qortal.arbitrary.patch.UnifiedDiffPatch;
import org.qortal.repository.DataException;
import org.qortal.settings.Settings;
import org.qortal.utils.FilesystemUtils;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.*;
import java.util.List;
import java.util.Objects;
import java.util.UUID;
public class ArbitraryDataMerge {
@ -27,18 +22,16 @@ public class ArbitraryDataMerge {
private Path pathBefore;
private Path pathAfter;
private String patchType;
private Path mergePath;
private String identifier;
private ArbitraryDataMetadataPatch metadata;
public ArbitraryDataMerge(Path pathBefore, Path pathAfter, String patchType) {
public ArbitraryDataMerge(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
this.patchType = patchType;
}
public void compute() throws IOException {
public void compute() throws IOException, DataException {
try {
this.preExecute();
this.copyPreviousStateToMergePath();
@ -85,7 +78,7 @@ public class ArbitraryDataMerge {
this.metadata.read();
}
private void applyDifferences() throws IOException {
private void applyDifferences() throws IOException, DataException {
List<Path> addedPaths = this.metadata.getAddedPaths();
for (Path path : addedPaths) {
@ -94,10 +87,10 @@ public class ArbitraryDataMerge {
ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path);
}
List<Path> modifiedPaths = this.metadata.getModifiedPaths();
for (Path path : modifiedPaths) {
LOGGER.info("File was modified: {}", path.toString());
this.applyPatch(path);
List<ModifiedPath> modifiedPaths = this.metadata.getModifiedPaths();
for (ModifiedPath modifiedPath : modifiedPaths) {
LOGGER.info("File was modified: {}", modifiedPath.toString());
this.applyPatch(modifiedPath);
}
List<Path> removedPaths = this.metadata.getRemovedPaths();
@ -107,55 +100,19 @@ public class ArbitraryDataMerge {
}
}
private void applyPatch(Path path) throws IOException {
if (Objects.equals(this.patchType, "unified-diff")) {
private void applyPatch(ModifiedPath modifiedPath) throws IOException, DataException {
if (modifiedPath.getDiffType() == DiffType.UNIFIED_DIFF) {
// Create destination file from patch
this.applyUnifiedDiffPatch(path);
UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(pathBefore, pathAfter, mergePath);
unifiedDiffPatch.apply(modifiedPath.getPath());
}
else if (modifiedPath.getDiffType() == DiffType.COMPLETE_FILE) {
// Copy complete file
Path filePath = Paths.get(this.pathAfter.toString(), modifiedPath.getPath().toString());
ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, modifiedPath.getPath());
}
else {
// Copy complete file
Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
ArbitraryDataMerge.copyPathToBaseDir(filePath, this.mergePath, path);
}
}
private void applyUnifiedDiffPatch(Path path) throws IOException {
Path originalPath = Paths.get(this.pathBefore.toString(), path.toString());
Path patchPath = Paths.get(this.pathAfter.toString(), path.toString());
Path mergePath = Paths.get(this.mergePath.toString(), path.toString());
if (!patchPath.toFile().exists()) {
throw new IllegalStateException("Patch file doesn't exist, but its path was included in modifiedPaths");
}
// Delete an existing file, as we are starting from a duplicate of pathBefore
File destFile = mergePath.toFile();
if (destFile.exists() && destFile.isFile()) {
Files.delete(mergePath);
}
List<String> originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8);
List<String> patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8);
// At first, parse the unified diff file and get the patch
Patch<String> patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents);
// Then apply the computed patch to the given text
try {
List<String> patchedContents = DiffUtils.patch(originalContents, patch);
// Write the patched file to the merge directory
FileWriter fileWriter = new FileWriter(mergePath.toString(), true);
BufferedWriter writer = new BufferedWriter(fileWriter);
for (String line : patchedContents) {
writer.append(line);
writer.newLine();
}
writer.flush();
writer.close();
} catch (PatchFailedException e) {
throw new IllegalStateException(String.format("Failed to apply patch for path %s: %s", path, e.getMessage()));
throw new DataException(String.format("Unrecognized patch diff type: %s", modifiedPath.getDiffType()));
}
}

40
src/main/java/org/qortal/arbitrary/metadata/ArbitraryDataMetadataPatch.java
View File

@ -4,6 +4,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.json.JSONArray;
import org.json.JSONObject;
import org.qortal.arbitrary.ArbitraryDataDiff.*;
import org.qortal.utils.Base58;
import java.lang.reflect.Field;
@ -17,9 +18,8 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataMetadataPatch.class);
private String patchType;
private List<Path> addedPaths;
private List<Path> modifiedPaths;
private List<ModifiedPath> modifiedPaths;
private List<Path> removedPaths;
private byte[] previousSignature;
private byte[] previousHash;
@ -44,12 +44,6 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
}
JSONObject patch = new JSONObject(this.jsonString);
if (patch.has("patchType")) {
String patchType = patch.getString("patchType");
if (patchType != null) {
this.patchType = patchType;
}
}
if (patch.has("prevSig")) {
String prevSig = patch.getString("prevSig");
if (prevSig != null) {
@ -75,8 +69,9 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
JSONArray modified = (JSONArray) patch.get("modified");
if (modified != null) {
for (int i=0; i<modified.length(); i++) {
String pathString = modified.getString(i);
this.modifiedPaths.add(Paths.get(pathString));
JSONObject jsonObject = modified.getJSONObject(i);
ModifiedPath modifiedPath = new ModifiedPath(jsonObject);
this.modifiedPaths.add(modifiedPath);
}
}
}
@ -103,26 +98,25 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
} catch (IllegalAccessException | NoSuchFieldException e) {
// Don't worry about failures as this is for optional ordering only
}
patch.put("patchType", this.patchType);
patch.put("prevSig", Base58.encode(this.previousSignature));
patch.put("prevHash", Base58.encode(this.previousHash));
patch.put("added", new JSONArray(this.addedPaths));
patch.put("modified", new JSONArray(this.modifiedPaths));
patch.put("removed", new JSONArray(this.removedPaths));
JSONArray modifiedPaths = new JSONArray();
for (ModifiedPath modifiedPath : this.modifiedPaths) {
JSONObject modifiedPathJson = new JSONObject();
modifiedPathJson.put("path", modifiedPath.getPath());
modifiedPathJson.put("type", modifiedPath.getDiffType());
modifiedPaths.put(modifiedPathJson);
}
patch.put("modified", modifiedPaths);
this.jsonString = patch.toString(2);
LOGGER.info("Patch metadata: {}", this.jsonString);
}
public void setPatchType(String patchType) {
this.patchType = patchType;
}
public String getPatchType() {
return this.patchType;
}
public void setAddedPaths(List<Path> addedPaths) {
this.addedPaths = addedPaths;
}
@ -131,11 +125,11 @@ public class ArbitraryDataMetadataPatch extends ArbitraryDataMetadata {
return this.addedPaths;
}
public void setModifiedPaths(List<Path> modifiedPaths) {
public void setModifiedPaths(List<ModifiedPath> modifiedPaths) {
this.modifiedPaths = modifiedPaths;
}
public List<Path> getModifiedPaths() {
public List<ModifiedPath> getModifiedPaths() {
return this.modifiedPaths;
}

208
src/main/java/org/qortal/arbitrary/patch/UnifiedDiffPatch.java Normal file
View File

@ -0,0 +1,208 @@
package org.qortal.arbitrary.patch;
import com.github.difflib.DiffUtils;
import com.github.difflib.UnifiedDiffUtils;
import com.github.difflib.patch.Patch;
import com.github.difflib.patch.PatchFailedException;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.Crypto;
import org.qortal.settings.Settings;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
public class UnifiedDiffPatch {
private static final Logger LOGGER = LogManager.getLogger(UnifiedDiffPatch.class);
private Path before;
private Path after;
private Path destination;
private String identifier;
private Path validationPath;
public UnifiedDiffPatch(Path before, Path after, Path destination) {
this.before = before;
this.after = after;
this.destination = destination;
}
/**
* Create a patch based on the differences in path "after"
* compared with base path "before", outputting the patch
* to the "destination" path.
*
* @throws IOException
*/
public void create() throws IOException {
if (!Files.exists(before)) {
throw new IOException(String.format("File not found (before): %s", before.toString()));
}
if (!Files.exists(after)) {
throw new IOException(String.format("File not found (after): %s", after.toString()));
}
// Ensure parent folders exist in the destination
File file = new File(destination.toString());
File parent = file.getParentFile();
if (parent != null) {
parent.mkdirs();
}
// Delete an existing file if it exists
File destFile = destination.toFile();
if (destFile.exists() && destFile.isFile()) {
Files.delete(destination);
}
// Load the two files into memory
List<String> original = FileUtils.readLines(before.toFile(), StandardCharsets.UTF_8);
List<String> revised = FileUtils.readLines(after.toFile(), StandardCharsets.UTF_8);
// Generate diff information
Patch<String> diff = DiffUtils.diff(original, revised);
// Generate unified diff format
String originalFileName = before.getFileName().toString();
String revisedFileName = after.getFileName().toString();
List<String> unifiedDiff = UnifiedDiffUtils.generateUnifiedDiff(originalFileName, revisedFileName, original, diff, 0);
// Write the diff to the destination directory
FileWriter fileWriter = new FileWriter(destination.toString(), true);
BufferedWriter writer = new BufferedWriter(fileWriter);
for (String line : unifiedDiff) {
writer.append(line);
writer.newLine();
}
writer.flush();
writer.close();
}
/**
* Validate the patch to ensure it works correctly
*
* @return true if valid, false if invalid
* @throws IOException
*/
public boolean isValid() {
this.createRandomIdentifier();
this.createTempValidationDirectory();
// Merge the patch with the original path
Path tempPath = Paths.get(this.validationPath.toString(), this.identifier);
try {
UnifiedDiffPatch unifiedDiffPatch = new UnifiedDiffPatch(before, destination, tempPath);
unifiedDiffPatch.apply(null);
byte[] inputDigest = Crypto.digest(after.toFile());
byte[] outputDigest = Crypto.digest(tempPath.toFile());
if (Arrays.equals(inputDigest, outputDigest)) {
// Patch is valid
return true;
}
else {
LOGGER.info("Checksum mismatch when verifying patch for file {}", destination.toString());
return false;
}
}
catch (IOException e) {
LOGGER.info("Failed to compute merge for file {}: {}", destination.toString(), e.getMessage());
}
finally {
try {
Files.delete(tempPath);
} catch (IOException e) {
// Not important - will be cleaned up later
}
}
return false;
}
/**
* Apply a patch at path "after" on top of base path "before",
* outputting the combined results to the "destination" path.
* If before and after are directories, a relative path suffix
* can be used to specify the file within these folder structures.
*
* @param pathSuffix - a file path to append to the base paths, or null if the base paths are already files
* @throws IOException
*/
public void apply(Path pathSuffix) throws IOException {
Path originalPath = this.before;
Path patchPath = this.after;
Path mergePath = this.destination;
// If a path has been supplied, we need to append it to the base paths
if (pathSuffix != null) {
originalPath = Paths.get(this.before.toString(), pathSuffix.toString());
patchPath = Paths.get(this.after.toString(), pathSuffix.toString());
mergePath = Paths.get(this.destination.toString(), pathSuffix.toString());
}
if (!patchPath.toFile().exists()) {
throw new IllegalStateException("Patch file doesn't exist, but its path was included in modifiedPaths");
}
// Delete an existing file, as we are starting from a duplicate of pathBefore
File destFile = mergePath.toFile();
if (destFile.exists() && destFile.isFile()) {
Files.delete(mergePath);
}
List<String> originalContents = FileUtils.readLines(originalPath.toFile(), StandardCharsets.UTF_8);
List<String> patchContents = FileUtils.readLines(patchPath.toFile(), StandardCharsets.UTF_8);
// At first, parse the unified diff file and get the patch
Patch<String> patch = UnifiedDiffUtils.parseUnifiedDiff(patchContents);
// Then apply the computed patch to the given text
try {
List<String> patchedContents = DiffUtils.patch(originalContents, patch);
// Write the patched file to the merge directory
FileWriter fileWriter = new FileWriter(mergePath.toString(), true);
BufferedWriter writer = new BufferedWriter(fileWriter);
for (String line : patchedContents) {
writer.append(line);
writer.newLine();
}
writer.flush();
writer.close();
} catch (PatchFailedException e) {
throw new IllegalStateException(String.format("Failed to apply patch for path %s: %s", pathSuffix, e.getMessage()));
}
}
private void createRandomIdentifier() {
this.identifier = UUID.randomUUID().toString();
}
private void createTempValidationDirectory() {
// Use the user-specified temp dir, as it is deterministic, and is more likely to be located on reusable storage hardware
String baseDir = Settings.getInstance().getTempDataPath();
Path tempDir = Paths.get(baseDir, "diff", "validate");
try {
Files.createDirectories(tempDir);
} catch (IOException e) {
throw new IllegalStateException("Unable to create temp directory");
}
this.validationPath = tempDir;
}
}

282
src/test/java/org/qortal/test/arbitrary/ArbitraryDataMergeTests.java
View File

@ -9,11 +9,17 @@ import org.qortal.crypto.Crypto;
import org.qortal.repository.DataException;
import org.qortal.test.common.Common;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Objects;
import java.util.Random;
import static org.junit.Assert.*;
@ -81,7 +87,7 @@ public class ArbitraryDataMergeTests extends Common {
));
// Now merge the patch with the original path
ArbitraryDataMerge merge = new ArbitraryDataMerge(path1, patchPath, "unified-diff");
ArbitraryDataMerge merge = new ArbitraryDataMerge(path1, patchPath);
merge.compute();
Path finalPath = merge.getMergePath();
@ -117,7 +123,7 @@ public class ArbitraryDataMergeTests extends Common {
// Also check that the directory digests match
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(path2);
path2Digest.compute();
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(path2);
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath);
finalPathDigest.compute();
assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58());
}
@ -140,4 +146,276 @@ public class ArbitraryDataMergeTests extends Common {
}
@Test
public void testMergeBinaryFiles() throws IOException, DataException {
// Create two files in random temp directories
Path tempDir1 = Files.createTempDirectory("testMergeBinaryFiles1");
Path tempDir2 = Files.createTempDirectory("testMergeBinaryFiles2");
File file1 = new File(Paths.get(tempDir1.toString(), "file.bin").toString());
File file2 = new File(Paths.get(tempDir2.toString(), "file.bin").toString());
file1.deleteOnExit();
file2.deleteOnExit();
// Write random data to the first file
byte[] initialData = new byte[1024];
new Random().nextBytes(initialData);
Files.write(file1.toPath(), initialData);
byte[] file1Digest = Crypto.digest(file1);
// Write slightly modified data to the second file (bytes 100-116 are zeroed out)
byte[] updatedData = Arrays.copyOf(initialData, initialData.length);
final ByteBuffer byteBuffer = ByteBuffer.wrap(updatedData);
byteBuffer.position(100);
byteBuffer.put(new byte[16]);
updatedData = byteBuffer.array();
Files.write(file2.toPath(), updatedData);
byte[] file2Digest = Crypto.digest(file2);
// Make sure the two arrays are different
assertFalse(Arrays.equals(initialData, updatedData));
// And double check that they are both 1024 bytes long
assertEquals(1024, initialData.length);
assertEquals(1024, updatedData.length);
// Ensure both files exist
assertTrue(Files.exists(file1.toPath()));
assertTrue(Files.exists(file2.toPath()));
// Create a patch from the two paths
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]);
patch.create();
Path patchPath = patch.getFinalPath();
assertTrue(Files.exists(patchPath));
// Check that the patch file exists
Path patchFilePath = Paths.get(patchPath.toString(), "file.bin");
assertTrue(Files.exists(patchFilePath));
byte[] patchDigest = Crypto.digest(patchFilePath.toFile());
// Ensure that the patch file matches file2 exactly
// This is because binary files cannot currently be patched, and so the complete file
// is included instead
assertArrayEquals(patchDigest, file2Digest);
// Make sure that the patch file is different from file1
assertFalse(Arrays.equals(patchDigest, file1Digest));
// Now merge the patch with the original path
ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath);
merge.compute();
Path finalPath = merge.getMergePath();
// Check that the directory digests match
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2);
path2Digest.compute();
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath);
finalPathDigest.compute();
assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58());
}
@Test
public void testMergeRandomStrings() throws IOException, DataException {
// Create two files in random temp directories
Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings");
Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings");
File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString());
File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString());
file1.deleteOnExit();
file2.deleteOnExit();
// Write a random string to the first file
BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1));
String initialString = this.generateRandomString(1024);
file1Writer.write(initialString);
file1Writer.newLine();
file1Writer.close();
byte[] file1Digest = Crypto.digest(file1);
// Write a slightly modified string to the second file
BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2));
String updatedString = initialString.concat("-edit");
file2Writer.write(updatedString);
file2Writer.newLine();
file2Writer.close();
byte[] file2Digest = Crypto.digest(file2);
// Make sure the two strings are different
assertFalse(Objects.equals(initialString, updatedString));
// Ensure both files exist
assertTrue(Files.exists(file1.toPath()));
assertTrue(Files.exists(file2.toPath()));
// Create a patch from the two paths
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]);
patch.create();
Path patchPath = patch.getFinalPath();
assertTrue(Files.exists(patchPath));
// Check that the patch file exists
Path patchFilePath = Paths.get(patchPath.toString(), "file.txt");
assertTrue(Files.exists(patchFilePath));
byte[] patchDigest = Crypto.digest(patchFilePath.toFile());
// Make sure that the patch file is different from file1 and file2
assertFalse(Arrays.equals(patchDigest, file1Digest));
assertFalse(Arrays.equals(patchDigest, file2Digest));
// Now merge the patch with the original path
ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath);
merge.compute();
Path finalPath = merge.getMergePath();
// Check that the directory digests match
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2);
path2Digest.compute();
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath);
finalPathDigest.compute();
assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58());
}
@Test
public void testMergeRandomStringsWithoutTrailingNewlines() throws IOException, DataException {
// Create two files in random temp directories
Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings");
Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings");
File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString());
File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString());
file1.deleteOnExit();
file2.deleteOnExit();
// Write a random string to the first file
BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1));
String initialString = this.generateRandomString(1024);
file1Writer.write(initialString);
// No newline
file1Writer.close();
byte[] file1Digest = Crypto.digest(file1);
// Write a slightly modified string to the second file
BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2));
String updatedString = initialString.concat("-edit");
file2Writer.write(updatedString);
// No newline
file2Writer.close();
byte[] file2Digest = Crypto.digest(file2);
// Make sure the two strings are different
assertFalse(Objects.equals(initialString, updatedString));
// Ensure both files exist
assertTrue(Files.exists(file1.toPath()));
assertTrue(Files.exists(file2.toPath()));
// Create a patch from the two paths
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]);
patch.create();
Path patchPath = patch.getFinalPath();
assertTrue(Files.exists(patchPath));
// Check that the patch file exists
Path patchFilePath = Paths.get(patchPath.toString(), "file.txt");
assertTrue(Files.exists(patchFilePath));
byte[] patchDigest = Crypto.digest(patchFilePath.toFile());
// The patch file should be identical to file2, because we don't currently
// support arbitrary diff patches on files without trailing newlines
assertArrayEquals(patchDigest, file2Digest);
// Make sure that the patch file is different from file1
assertFalse(Arrays.equals(patchDigest, file1Digest));
// Now merge the patch with the original path
ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath);
merge.compute();
Path finalPath = merge.getMergePath();
// Check that the directory digests match
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2);
path2Digest.compute();
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath);
finalPathDigest.compute();
assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58());
}
@Test
public void testMergeRandomLargeStrings() throws IOException, DataException {
// Create two files in random temp directories
Path tempDir1 = Files.createTempDirectory("testMergeRandomStrings");
Path tempDir2 = Files.createTempDirectory("testMergeRandomStrings");
File file1 = new File(Paths.get(tempDir1.toString(), "file.txt").toString());
File file2 = new File(Paths.get(tempDir2.toString(), "file.txt").toString());
file1.deleteOnExit();
file2.deleteOnExit();
// Write a random string to the first file
BufferedWriter file1Writer = new BufferedWriter(new FileWriter(file1));
String initialString = this.generateRandomString(110 * 1024);
file1Writer.write(initialString);
file1Writer.newLine();
file1Writer.close();
byte[] file1Digest = Crypto.digest(file1);
// Write a slightly modified string to the second file
BufferedWriter file2Writer = new BufferedWriter(new FileWriter(file2));
String updatedString = initialString.concat("-edit");
file2Writer.write(updatedString);
file2Writer.newLine();
file2Writer.close();
byte[] file2Digest = Crypto.digest(file2);
// Make sure the two strings are different
assertFalse(Objects.equals(initialString, updatedString));
// Ensure both files exist
assertTrue(Files.exists(file1.toPath()));
assertTrue(Files.exists(file2.toPath()));
// Create a patch from the two paths
ArbitraryDataCreatePatch patch = new ArbitraryDataCreatePatch(tempDir1, tempDir2, new byte[16]);
patch.create();
Path patchPath = patch.getFinalPath();
assertTrue(Files.exists(patchPath));
// Check that the patch file exists
Path patchFilePath = Paths.get(patchPath.toString(), "file.txt");
assertTrue(Files.exists(patchFilePath));
byte[] patchDigest = Crypto.digest(patchFilePath.toFile());
// The patch file should be identical to file2 because the source files
// were over the maximum size limit for creating patches
assertArrayEquals(patchDigest, file2Digest);
// Make sure that the patch file is different from file1
assertFalse(Arrays.equals(patchDigest, file1Digest));
// Now merge the patch with the original path
ArbitraryDataMerge merge = new ArbitraryDataMerge(tempDir1, patchPath);
merge.compute();
Path finalPath = merge.getMergePath();
// Check that the directory digests match
ArbitraryDataDigest path2Digest = new ArbitraryDataDigest(tempDir2);
path2Digest.compute();
ArbitraryDataDigest finalPathDigest = new ArbitraryDataDigest(finalPath);
finalPathDigest.compute();
assertEquals(path2Digest.getHash58(), finalPathDigest.getHash58());
}
private String generateRandomString(int length) {
int leftLimit = 48; // numeral '0'
int rightLimit = 122; // letter 'z'
Random random = new Random();
return random.ints(leftLimit, rightLimit + 1)
.filter(i -> (i <= 57 || i >= 65) && (i <= 90 || i >= 97))
.limit(length)
.collect(StringBuilder::new, StringBuilder::appendCodePoint, StringBuilder::append)
.toString();
}
}

1
src/test/resources/arbitrary/demo1/.qortal/cache
View File

@ -1 +0,0 @@
db2d9ab2-a97e-43bf-a259-ebbc1a1b0c59