crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Use the /.qortal/patch file to apply differences when merging together two layers, rather than walking through the file tree like we did before.

Browse Source
This commit is contained in:
CalDescent 2021-08-15 18:49:30 +01:00
parent fa696a2901
commit 8fac0a02e5
4 changed files with 157 additions and 108 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main/java/org/qortal/arbitrary/ArbitraryDataDiff.java
View File

@ -200,8 +200,11 @@ public class ArbitraryDataDiff {
}
private void writeMetadata() throws IOException {
ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.addedPaths, this.modifiedPaths,
this.removedPaths, this.diffPath, this.previousSignature);
ArbitraryDataMetadata metadata = new ArbitraryDataMetadata(this.diffPath);
metadata.setAddedPaths(this.addedPaths);
metadata.setModifiedPaths(this.modifiedPaths);
metadata.setRemovedPaths(this.removedPaths);
metadata.setPreviousSignature(this.previousSignature);
metadata.write();
}

112
src/main/java/org/qortal/arbitrary/ArbitraryDataMerge.java
View File

@ -3,15 +3,13 @@ package org.qortal.arbitrary;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.Crypto;
import org.qortal.settings.Settings;
import org.qortal.utils.FilesystemUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
public class ArbitraryDataMerge {
@ -22,6 +20,7 @@ public class ArbitraryDataMerge {
private Path pathAfter;
private Path mergePath;
private String identifier;
private ArbitraryDataMetadata metadata;
public ArbitraryDataMerge(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
@ -32,7 +31,8 @@ public class ArbitraryDataMerge {
try {
this.preExecute();
this.copyPreviousStateToMergePath();
this.findDifferences();
this.loadMetadata();
this.applyDifferences();
} finally {
this.postExecute();
@ -68,97 +68,35 @@ public class ArbitraryDataMerge {
ArbitraryDataMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get(""));
}
private void findDifferences() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path mergePathAbsolute = this.mergePath.toAbsolutePath();
private void loadMetadata() throws IOException {
this.metadata = new ArbitraryDataMetadata(this.pathAfter);
this.metadata.read();
}
// LOGGER.info("this.pathBefore: {}", this.pathBefore);
// LOGGER.info("this.pathAfter: {}", this.pathAfter);
// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
// LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute);
private void applyDifferences() throws IOException {
List<Path> addedPaths = this.metadata.getAddedPaths();
for (Path path : addedPaths) {
LOGGER.info("File was added: {}", path.toString());
Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path);
}
try {
// Check for additions or modifications
Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
List<Path> modifiedPaths = this.metadata.getModifiedPaths();
for (Path path : modifiedPaths) {
LOGGER.info("File was modified: {}", path.toString());
Path filePath = Paths.get(this.pathAfter.toString(), path.toString());
ArbitraryDataMerge.copyFilePathToBaseDir(filePath, this.mergePath, path);
}
@Override
public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException {
Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath());
Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter);
boolean wasAdded = false;
boolean wasModified = false;
boolean wasRemoved = false;
if (after.toString().endsWith(".removed")) {
LOGGER.trace("File was removed: {}", after.toString());
wasRemoved = true;
}
else if (!Files.exists(filePathBefore)) {
LOGGER.trace("File was added: {}", after.toString());
wasAdded = true;
}
else if (Files.size(after) != Files.size(filePathBefore)) {
// Check file size first because it's quicker
LOGGER.trace("File size was modified: {}", after.toString());
wasModified = true;
}
else if (!Arrays.equals(ArbitraryDataMerge.digestFromPath(after), ArbitraryDataMerge.digestFromPath(filePathBefore))) {
// Check hashes as a last resort
LOGGER.trace("File contents were modified: {}", after.toString());
wasModified = true;
}
if (wasAdded | wasModified) {
ArbitraryDataMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter);
}
if (wasRemoved) {
if (filePathAfter.toString().endsWith(".removed")) {
// Trim the ".removed"
Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8));
ArbitraryDataMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed);
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException e){
LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
// TODO: throw exception?
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) {
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
List<Path> removedPaths = this.metadata.getRemovedPaths();
for (Path path : removedPaths) {
LOGGER.info("File was removed: {}", path.toString());
ArbitraryDataMerge.deletePathInBaseDir(this.mergePath, path);
}
}
private static byte[] digestFromPath(Path path) {
try {
return Crypto.digest(Files.readAllBytes(path));
} catch (IOException e) {
return null;
}
}
private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
if (!Files.exists(source)) {
throw new IOException(String.format("File not found: %s", source.toString()));

125
src/main/java/org/qortal/arbitrary/ArbitraryDataMetadata.java
View File

@ -7,11 +7,13 @@ import org.json.JSONObject;
import org.qortal.utils.Base58;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
public class ArbitraryDataMetadata {
@ -27,13 +29,18 @@ public class ArbitraryDataMetadata {
private String jsonString;
public ArbitraryDataMetadata(List<Path> addedPaths, List<Path> modifiedPaths, List<Path> removedPaths,
Path filePath, byte[] previousSignature) {
this.addedPaths = addedPaths;
this.modifiedPaths = modifiedPaths;
this.removedPaths = removedPaths;
public ArbitraryDataMetadata(Path filePath) {
this.filePath = filePath;
this.previousSignature = previousSignature;
this.qortalDirectoryPath = Paths.get(filePath.toString(), ".qortal");
this.addedPaths = new ArrayList<>();
this.modifiedPaths = new ArrayList<>();
this.removedPaths = new ArrayList<>();
}
public void read() throws IOException {
this.loadJson();
this.readJson();
}
public void write() throws IOException {
@ -42,39 +49,123 @@ public class ArbitraryDataMetadata {
this.writeToQortalPath();
}
private void loadJson() throws IOException {
Path path = Paths.get(this.qortalDirectoryPath.toString(), "patch");
File patchFile = new File(path.toString());
if (!patchFile.exists()) {
throw new IOException(String.format("Patch file doesn't exist: %s", path.toString()));
}
this.jsonString = new String(Files.readAllBytes(path));
}
private void readJson() {
if (this.jsonString == null) {
throw new IllegalStateException("Patch JSON string is null");
}
JSONObject patch = new JSONObject(this.jsonString);
if (patch.has("prevSig")) {
String prevSig = (String)patch.get("prevSig");
if (prevSig != null) {
this.previousSignature = Base58.decode(prevSig);
}
}
if (patch.has("added")) {
JSONArray added = (JSONArray) patch.get("added");
if (added != null) {
for (int i=0; i<added.length(); i++) {
String pathString = (String)added.get(i);
this.addedPaths.add(Paths.get(pathString));
}
}
}
if (patch.has("modified")) {
JSONArray modified = (JSONArray) patch.get("modified");
if (modified != null) {
for (int i=0; i<modified.length(); i++) {
String pathString = (String)modified.get(i);
this.modifiedPaths.add(Paths.get(pathString));
}
}
}
if (patch.has("removed")) {
JSONArray removed = (JSONArray) patch.get("removed");
if (removed != null) {
for (int i=0; i<removed.length(); i++) {
String pathString = (String)removed.get(i);
this.removedPaths.add(Paths.get(pathString));
}
}
}
}
private void buildJson() {
JSONArray addedPathsJson = new JSONArray(this.addedPaths);
JSONArray modifiedPathsJson = new JSONArray(this.modifiedPaths);
JSONArray removedPathsJson = new JSONArray(this.removedPaths);
String previousSignature58 = Base58.encode(this.previousSignature);
JSONObject jsonObject = new JSONObject();
jsonObject.put("prevSig", previousSignature58);
jsonObject.put("added", addedPathsJson);
jsonObject.put("modified", modifiedPathsJson);
jsonObject.put("removed", removedPathsJson);
JSONObject patch = new JSONObject();
patch.put("prevSig", previousSignature58);
patch.put("added", addedPathsJson);
patch.put("modified", modifiedPathsJson);
patch.put("removed", removedPathsJson);
this.jsonString = jsonObject.toString(4);
this.jsonString = patch.toString(2);
}
private void createQortalDirectory() {
Path qortalDir = Paths.get(this.filePath.toString(), ".qortal");
try {
Files.createDirectories(qortalDir);
Files.createDirectories(this.qortalDirectoryPath);
} catch (IOException e) {
throw new IllegalStateException("Unable to create .qortal directory");
}
this.qortalDirectoryPath = qortalDir;
}
private void writeToQortalPath() throws IOException {
Path statePath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
BufferedWriter writer = new BufferedWriter(new FileWriter(statePath.toString()));
Path patchPath = Paths.get(this.qortalDirectoryPath.toString(), "patch");
BufferedWriter writer = new BufferedWriter(new FileWriter(patchPath.toString()));
writer.write(this.jsonString);
writer.close();
}
public void setAddedPaths(List<Path> addedPaths) {
this.addedPaths = addedPaths;
}
public List<Path> getAddedPaths() {
return this.addedPaths;
}
public void setModifiedPaths(List<Path> modifiedPaths) {
this.modifiedPaths = modifiedPaths;
}
public List<Path> getModifiedPaths() {
return this.modifiedPaths;
}
public void setRemovedPaths(List<Path> removedPaths) {
this.removedPaths = removedPaths;
}
public List<Path> getRemovedPaths() {
return this.removedPaths;
}
public void setPreviousSignature(byte[] previousSignature) {
this.previousSignature = previousSignature;
}
public byte[] getPreviousSignature() {
return this.getPreviousSignature();
}
public String getJsonString() {
return this.jsonString;
}

21
src/main/java/org/qortal/arbitrary/ArbitraryDataWriter.java
View File

@ -133,11 +133,28 @@ public class ArbitraryDataWriter {
}
// Validate the patch
this.validatePatch();
}
private void validatePatch() throws IOException {
if (this.filePath == null) {
throw new IllegalStateException("Null path after creating patch");
}
if (FilesystemUtils.isDirectoryEmpty(this.filePath)) {
throw new IllegalStateException("Patch has no content. Either no files have changed, or something went wrong");
File qortalMetadataDirectoryFile = Paths.get(this.filePath.toString(), ".qortal").toFile();
if (!qortalMetadataDirectoryFile.exists()) {
throw new IllegalStateException("Qortal metadata folder doesn't exist in patch");
}
if (!qortalMetadataDirectoryFile.isDirectory()) {
throw new IllegalStateException("Qortal metadata folder isn't a directory");
}
File qortalPatchMetadataFile = Paths.get(this.filePath.toString(), ".qortal", "patch").toFile();
if (!qortalPatchMetadataFile.exists()) {
throw new IllegalStateException("Qortal patch metadata file doesn't exist in patch");
}
if (!qortalPatchMetadataFile.isFile()) {
throw new IllegalStateException("Qortal patch metadata file isn't a file");
}
}