AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Initial implementation of data patches/updates

Browse Source 
This adds support for the PATCH method in addition to the existing PUT method.

Currently, a patch includes only files that have been added or modified, as well as placeholder files to indicate those that have been removed.

This is not production ready, as I am hoping to create patches on a more granular level - i.e. just the modified bytes of each file. It would also make sense to track deletions using a metadata/manifest file in a hidden folder.

It also adds early support of accessing files using a name rather than a signature or hash.
This commit is contained in:
CalDescent 2021-08-14 13:11:36 +01:00
parent 5ac9e3e47a
commit e15cf063c6
17 changed files with 1124 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/main/java/org/qortal/api/HTMLParser.java
View File

@ -13,9 +13,9 @@ public class HTMLParser {
private String linkPrefix;
public HTMLParser(String resourceId, String inPath, boolean usePrefix) {
public HTMLParser(String resourceId, String inPath, String prefix, boolean usePrefix) {
String inPathWithoutFilename = inPath.substring(0, inPath.lastIndexOf('/'));
this.linkPrefix = usePrefix ? String.format("/site/%s%s", resourceId, inPathWithoutFilename) : "";
this.linkPrefix = usePrefix ? String.format("%s/%s%s", prefix, resourceId, inPathWithoutFilename) : "";
}
/**

4
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -272,10 +272,10 @@ public class ArbitraryResource {
Service service = Service.ARBITRARY_DATA;
Compression compression = Compression.NONE;
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression);
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression);
try {
dataFileWriter.save();
} catch (IOException e) {
} catch (IOException | DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
} catch (IllegalStateException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);

49
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@ -98,15 +98,15 @@ public class WebsiteResource {
}
byte[] creatorPublicKey = Base58.decode(creatorPublicKeyBase58);
String name = null;
ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PUT;
String name = "CalDescentTest1"; // TODO: dynamic
ArbitraryTransactionData.Method method = ArbitraryTransactionData.Method.PATCH;
ArbitraryTransactionData.Service service = ArbitraryTransactionData.Service.WEBSITE;
ArbitraryTransactionData.Compression compression = ArbitraryTransactionData.Compression.ZIP;
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), method, compression);
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(path), name, service, method, compression);
try {
dataFileWriter.save();
} catch (IOException e) {
} catch (IOException | DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
} catch (IllegalStateException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
@ -144,6 +144,7 @@ public class WebsiteResource {
secret, compression, digest, dataType, chunkHashes, payments);
ArbitraryTransaction transaction = (ArbitraryTransaction) Transaction.fromData(repository, transactionData);
LOGGER.info("Computing nonce...");
transaction.computeNonce();
Transaction.ValidationResult result = transaction.isValidUnconfirmed();
@ -197,13 +198,15 @@ public class WebsiteResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.NON_PRODUCTION);
}
String name = null;
Service service = Service.WEBSITE;
Method method = Method.PUT;
Compression compression = Compression.ZIP;
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), method, compression);
DataFileWriter dataFileWriter = new DataFileWriter(Paths.get(directoryPath), name, service, method, compression);
try {
dataFileWriter.save();
} catch (IOException e) {
} catch (IOException | DataException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE);
} catch (IllegalStateException e) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
@ -222,26 +225,38 @@ public class WebsiteResource {
@GET
@Path("{signature}")
public HttpServletResponse getIndexBySignature(@PathParam("signature") String signature) {
return this.get(signature, ResourceIdType.SIGNATURE, "/", null,true);
return this.get(signature, ResourceIdType.SIGNATURE, "/", null, "/site", true);
}
@GET
@Path("{signature}/{path:.*}")
public HttpServletResponse getPathBySignature(@PathParam("signature") String signature, @PathParam("path") String inPath) {
return this.get(signature, ResourceIdType.SIGNATURE, inPath,null,true);
return this.get(signature, ResourceIdType.SIGNATURE, inPath,null, "/site", true);
}
@GET
@Path("/hash/{hash}")
public HttpServletResponse getIndexByHash(@PathParam("hash") String hash58, @QueryParam("secret") String secret58) {
return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58,true);
return this.get(hash58, ResourceIdType.FILE_HASH, "/", secret58, "/site/hash", true);
}
@GET
@Path("/name/{name}/{path:.*}")
public HttpServletResponse getPathByName(@PathParam("name") String name, @PathParam("path") String inPath) {
return this.get(name, ResourceIdType.NAME, inPath, null, "/site/name", true);
}
@GET
@Path("/name/{name}")
public HttpServletResponse getIndexByName(@PathParam("name") String name) {
return this.get(name, ResourceIdType.NAME, "/", null, "/site/name", true);
}
@GET
@Path("/hash/{hash}/{path:.*}")
public HttpServletResponse getPathByHash(@PathParam("hash") String hash58, @PathParam("path") String inPath,
@QueryParam("secret") String secret58) {
return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58,true);
return this.get(hash58, ResourceIdType.FILE_HASH, inPath, secret58, "/site/hash", true);
}
@GET
@ -259,19 +274,23 @@ public class WebsiteResource {
private HttpServletResponse getDomainMap(String inPath) {
Map<String, String> domainMap = Settings.getInstance().getSimpleDomainMap();
if (domainMap != null && domainMap.containsKey(request.getServerName())) {
return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, false);
return this.get(domainMap.get(request.getServerName()), ResourceIdType.SIGNATURE, inPath, null, "", false);
}
return this.get404Response();
}
private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58, boolean usePrefix) {
private HttpServletResponse get(String resourceId, ResourceIdType resourceIdType, String inPath, String secret58,
String prefix, boolean usePrefix) {
if (!inPath.startsWith(File.separator)) {
inPath = File.separator + inPath;
}
DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType);
Service service = Service.WEBSITE;
DataFileReader dataFileReader = new DataFileReader(resourceId, resourceIdType, service);
dataFileReader.setSecret58(secret58); // Optional, used for loading encrypted file hashes only
try {
// TODO: overwrite if new transaction arrives, to invalidate cache
// We could store the latest transaction signature in the extracted folder
dataFileReader.load(false);
} catch (Exception e) {
return this.get404Response();
@ -289,7 +308,7 @@ public class WebsiteResource {
if (HTMLParser.isHtmlFile(filename)) {
// HTML file - needs to be parsed
byte[] data = Files.readAllBytes(Paths.get(filePath)); // TODO: limit file size that can be read into memory
HTMLParser htmlParser = new HTMLParser(resourceId, inPath, usePrefix);
HTMLParser htmlParser = new HTMLParser(resourceId, inPath, prefix, usePrefix);
data = htmlParser.replaceRelativeLinks(filename, data);
response.setContentType(context.getMimeType(filename));
response.setContentLength(data.length);
@ -311,7 +330,7 @@ public class WebsiteResource {
}
return response;
} catch (FileNotFoundException | NoSuchFileException e) {
LOGGER.info("File not found at path: {}", unzippedPath);
LOGGER.info("Unable to serve file: {}", e.getMessage());
if (inPath.equals("/")) {
// Delete the unzipped folder if no index file was found
try {

2
src/main/java/org/qortal/controller/ArbitraryDataManager.java
View File

@ -368,7 +368,7 @@ public class ArbitraryDataManager extends Thread {
// Load file(s) and add any that exist to the list of hashes
DataFile dataFile = DataFile.fromHash(hash);
if (chunkHashes.length > 0) {
if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes);
for (DataFileChunk dataFileChunk : dataFile.getChunks()) {
if (dataFileChunk.exists()) {

7
src/main/java/org/qortal/repository/ArbitraryRepository.java
View File

@ -1,6 +1,9 @@
package org.qortal.repository;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import java.util.List;
public interface ArbitraryRepository {
@ -12,4 +15,8 @@ public interface ArbitraryRepository {
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException;
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException;
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException;
}

145
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -3,12 +3,20 @@ package org.qortal.repository.hsqldb;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.Crypto;
import org.qortal.data.PaymentData;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.DataType;
import org.qortal.repository.ArbitraryRepository;
import org.qortal.repository.DataException;
import org.qortal.storage.DataFile;
import org.qortal.transaction.Transaction.ApprovalStatus;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
public class HSQLDBArbitraryRepository implements ArbitraryRepository {
@ -48,7 +56,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
// Load data file(s)
DataFile dataFile = DataFile.fromHash(digest);
if (chunkHashes.length > 0) {
if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes);
}
@ -83,7 +91,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
// Load data file(s)
DataFile dataFile = DataFile.fromHash(digest);
if (chunkHashes.length > 0) {
if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes);
}
@ -168,7 +176,7 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
// Load data file(s)
DataFile dataFile = DataFile.fromHash(digest);
if (chunkHashes.length > 0) {
if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes);
}
@ -176,4 +184,133 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
dataFile.deleteAll();
}
@Override
public List<ArbitraryTransactionData> getArbitraryTransactions(String name, Service service, long since) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name = ? AND service = ? AND created_when >= ?" +
"ORDER BY created_when ASC";
List<ArbitraryTransactionData> arbitraryTransactionData = new ArrayList<>();
try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, since)) {
if (resultSet == null)
return null;
do {
//TransactionType type = TransactionType.valueOf(resultSet.getInt(1));
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
Service serviceResult = Service.valueOf(resultSet.getInt(13));
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] chunkHashes = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
Method method = Method.valueOf(resultSet.getInt(19));
byte[] secret = resultSet.getBytes(20);
Compression compression = Compression.valueOf(resultSet.getInt(21));
List<PaymentData> payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature());
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceResult, nonce, size, nameResult, method, secret, compression, data,
dataType, chunkHashes, payments);
arbitraryTransactionData.add(transactionData);
} while (resultSet.next());
return arbitraryTransactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
}
}
@Override
public ArbitraryTransactionData getLatestTransaction(String name, Service service, Method method) throws DataException {
String sql = "SELECT type, reference, signature, creator, created_when, fee, " +
"tx_group_id, block_height, approval_status, approval_height, " +
"version, nonce, service, size, is_data_raw, data, chunk_hashes, " +
"name, update_method, secret, compression FROM ArbitraryTransactions " +
"JOIN Transactions USING (signature) " +
"WHERE name = ? AND service = ? AND update_method = ? " +
"ORDER BY created_when DESC LIMIT 1";
try (ResultSet resultSet = this.repository.checkedExecute(sql, name, service.value, method.value)) {
if (resultSet == null)
return null;
//TransactionType type = TransactionType.valueOf(resultSet.getInt(1));
byte[] reference = resultSet.getBytes(2);
byte[] signature = resultSet.getBytes(3);
byte[] creatorPublicKey = resultSet.getBytes(4);
long timestamp = resultSet.getLong(5);
Long fee = resultSet.getLong(6);
if (fee == 0 && resultSet.wasNull())
fee = null;
int txGroupId = resultSet.getInt(7);
Integer blockHeight = resultSet.getInt(8);
if (blockHeight == 0 && resultSet.wasNull())
blockHeight = null;
ApprovalStatus approvalStatus = ApprovalStatus.valueOf(resultSet.getInt(9));
Integer approvalHeight = resultSet.getInt(10);
if (approvalHeight == 0 && resultSet.wasNull())
approvalHeight = null;
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, creatorPublicKey, fee, approvalStatus, blockHeight, approvalHeight, signature);
int version = resultSet.getInt(11);
int nonce = resultSet.getInt(12);
Service serviceResult = Service.valueOf(resultSet.getInt(13));
int size = resultSet.getInt(14);
boolean isDataRaw = resultSet.getBoolean(15); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(16);
byte[] chunkHashes = resultSet.getBytes(17);
String nameResult = resultSet.getString(18);
Method methodResult = Method.valueOf(resultSet.getInt(19));
byte[] secret = resultSet.getBytes(20);
Compression compression = Compression.valueOf(resultSet.getInt(21));
List<PaymentData> payments = new ArrayList<>(); // TODO: this.getPaymentsFromSignature(baseTransactionData.getSignature());
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
version, serviceResult, nonce, size, nameResult, methodResult, secret, compression, data,
dataType, chunkHashes, payments);
return transactionData;
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transactions from repository", e);
}
}
}

4
src/main/java/org/qortal/storage/DataFile.java
View File

@ -44,7 +44,9 @@ public class DataFile {
// Resource ID types
public enum ResourceIdType {
SIGNATURE,
FILE_HASH
FILE_HASH,
TRANSACTION_DATA,
NAME
};
private static final Logger LOGGER = LogManager.getLogger(DataFile.class);

131
src/main/java/org/qortal/storage/DataFileBuilder.java Normal file
View File

@ -0,0 +1,131 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.Method;
import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.storage.DataFile.ResourceIdType;
import org.qortal.utils.Base58;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
public class DataFileBuilder {
private static final Logger LOGGER = LogManager.getLogger(DataFileBuilder.class);
private String name;
private Service service;
private List<ArbitraryTransactionData> transactions;
private ArbitraryTransactionData latestPutTransaction;
private List<Path> paths;
private Path finalPath;
public DataFileBuilder(String name, Service service) {
this.name = name;
this.service = service;
this.paths = new ArrayList<>();
}
public void build() throws DataException, IOException {
this.fetchTransactions();
this.validateTransactions();
this.processTransactions();
this.buildLatestState();
}
private void fetchTransactions() throws DataException {
try (final Repository repository = RepositoryManager.getRepository()) {
// Get the most recent PUT
ArbitraryTransactionData latestPut = repository.getArbitraryRepository()
.getLatestTransaction(this.name, this.service, Method.PUT);
if (latestPut == null) {
throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first.");
}
this.latestPutTransaction = latestPut;
// Load all transactions since the latest PUT
List<ArbitraryTransactionData> transactionDataList = repository.getArbitraryRepository()
.getArbitraryTransactions(this.name, this.service, latestPut.getTimestamp());
this.transactions = transactionDataList;
}
}
private void validateTransactions() {
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
ArbitraryTransactionData latestPut = this.latestPutTransaction;
if (latestPut == null) {
throw new IllegalStateException("Cannot PATCH without existing PUT. Deploy using PUT first.");
}
if (latestPut.getMethod() != Method.PUT) {
throw new IllegalStateException("Expected PUT but received PATCH");
}
if (transactionDataList.size() == 0) {
throw new IllegalStateException(String.format("No transactions found for name %s, service %s, since %d",
name, service, latestPut.getTimestamp()));
}
// Verify that the signature of the first transaction matches the latest PUT
ArbitraryTransactionData firstTransaction = transactionDataList.get(0);
if (!Objects.equals(firstTransaction.getSignature(), latestPut.getSignature())) {
throw new IllegalStateException("First transaction did not match latest PUT transaction");
}
// Remove the first transaction, as it should be the only PUT
transactionDataList.remove(0);
for (ArbitraryTransactionData transactionData : transactionDataList) {
if (!(transactionData instanceof ArbitraryTransactionData)) {
String sig58 = Base58.encode(transactionData.getSignature());
throw new IllegalStateException(String.format("Received non-arbitrary transaction: %s", sig58));
}
if (transactionData.getMethod() != Method.PATCH) {
throw new IllegalStateException("Expected PATCH but received PUT");
}
}
}
private void processTransactions() throws IOException, DataException {
List<ArbitraryTransactionData> transactionDataList = new ArrayList<>(this.transactions);
for (ArbitraryTransactionData transactionData : transactionDataList) {
LOGGER.trace("Found arbitrary transaction {}", Base58.encode(transactionData.getSignature()));
// Build the data file, overwriting anything that was previously there
String sig58 = Base58.encode(transactionData.getSignature());
DataFileReader dataFileReader = new DataFileReader(sig58, ResourceIdType.TRANSACTION_DATA, this.service);
dataFileReader.setTransactionData(transactionData);
dataFileReader.load(true);
Path path = dataFileReader.getFilePath();
if (path == null) {
throw new IllegalStateException(String.format("Null path when building data from transaction %s", sig58));
}
if (!Files.exists(path)) {
throw new IllegalStateException(String.format("Path doesn't exist when building data from transaction %s", sig58));
}
paths.add(path);
}
}
private void buildLatestState() throws IOException, DataException {
DataFilePatches dataFilePatches = new DataFilePatches(this.paths);
dataFilePatches.applyPatches();
this.finalPath = dataFilePatches.getFinalPath();
}
public Path getFinalPath() {
return this.finalPath;
}
}

56
src/main/java/org/qortal/storage/DataFileCombiner.java Normal file
View File

@ -0,0 +1,56 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public class DataFileCombiner {
private static final Logger LOGGER = LogManager.getLogger(DataFileCombiner.class);
private Path pathBefore;
private Path pathAfter;
private Path finalPath;
public DataFileCombiner(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
}
public void combine() throws IOException {
try {
this.preExecute();
this.process();
} finally {
this.postExecute();
}
}
private void preExecute() {
if (this.pathBefore == null || this.pathAfter == null) {
throw new IllegalStateException(String.format("No paths available to build patch"));
}
if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist"));
}
}
private void postExecute() {
}
private void process() throws IOException {
DataFileMerge merge = new DataFileMerge(this.pathBefore, this.pathAfter);
merge.compute();
this.finalPath = merge.getMergePath();
}
public Path getFinalPath() {
return this.finalPath;
}
}

58
src/main/java/org/qortal/storage/DataFileCreatePatch.java Normal file
View File

@ -0,0 +1,58 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.repository.DataException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
public class DataFileCreatePatch {
private static final Logger LOGGER = LogManager.getLogger(DataFileCreatePatch.class);
private Path pathBefore;
private Path pathAfter;
private Path finalPath;
public DataFileCreatePatch(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
}
public void create() throws DataException, IOException {
try {
this.preExecute();
this.process();
} finally {
this.postExecute();
}
}
private void preExecute() {
if (this.pathBefore == null || this.pathAfter == null) {
throw new IllegalStateException(String.format("No paths available to build patch"));
}
if (!Files.exists(this.pathBefore) || !Files.exists(this.pathAfter)) {
throw new IllegalStateException(String.format("Unable to create patch because at least one path doesn't exist"));
}
}
private void postExecute() {
}
private void process() {
DataFileDiff diff = new DataFileDiff(this.pathBefore, this.pathAfter);
diff.compute();
this.finalPath = diff.getDiffPath();
}
public Path getFinalPath() {
return this.finalPath;
}
}

218
src/main/java/org/qortal/storage/DataFileDiff.java Normal file
View File

@ -0,0 +1,218 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.Crypto;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
public class DataFileDiff {
private static final Logger LOGGER = LogManager.getLogger(DataFileDiff.class);
private Path pathBefore;
private Path pathAfter;
private Path diffPath;
public DataFileDiff(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
}
public void compute() {
try {
this.preExecute();
this.findAddedOrModifiedFiles();
this.findRemovedFiles();
} finally {
this.postExecute();
}
}
private void preExecute() {
this.createOutputDirectory();
}
private void postExecute() {
}
private void createOutputDirectory() {
// Ensure temp folder exists
Path tempDir;
try {
tempDir = Files.createTempDirectory("qortal-diff");
} catch (IOException e) {
throw new IllegalStateException("Unable to create temp directory");
}
this.diffPath = tempDir;
}
private void findAddedOrModifiedFiles() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
// LOGGER.info("this.pathBefore: {}", this.pathBefore);
// LOGGER.info("this.pathAfter: {}", this.pathAfter);
// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
// LOGGER.info("diffPathAbsolute: {}", diffPathAbsolute);
try {
// Check for additions or modifications
Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException {
Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath());
Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter);
boolean wasAdded = false;
boolean wasModified = false;
if (!Files.exists(filePathBefore)) {
LOGGER.info("File was added: {}", after.toString());
wasAdded = true;
}
else if (Files.size(after) != Files.size(filePathBefore)) {
// Check file size first because it's quicker
LOGGER.info("File size was modified: {}", after.toString());
wasModified = true;
}
else if (!Arrays.equals(DataFileDiff.digestFromPath(after), DataFileDiff.digestFromPath(filePathBefore))) {
// Check hashes as a last resort
LOGGER.info("File contents were modified: {}", after.toString());
wasModified = true;
}
if (wasAdded | wasModified) {
DataFileDiff.copyFilePathToBaseDir(after, diffPathAbsolute, filePathAfter);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException e){
LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
// TODO: throw exception?
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) {
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
}
}
private void findRemovedFiles() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path diffPathAbsolute = this.diffPath.toAbsolutePath();
try {
// Check for removals
Files.walkFileTree(this.pathBefore, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path before, BasicFileAttributes attrs) throws IOException {
Path directoryPathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
Path directoryPathAfter = pathAfterAbsolute.resolve(directoryPathBefore);
if (!Files.exists(directoryPathAfter)) {
LOGGER.info("Directory was removed: {}", directoryPathAfter.toString());
DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, directoryPathBefore);
// TODO: we might need to mark directories differently to files
// TODO: add path to manifest JSON
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path before, BasicFileAttributes attrs) throws IOException {
Path filePathBefore = pathBeforeAbsolute.relativize(before.toAbsolutePath());
Path filePathAfter = pathAfterAbsolute.resolve(filePathBefore);
if (!Files.exists(filePathAfter)) {
LOGGER.trace("File was removed: {}", before.toString());
DataFileDiff.markFilePathAsRemoved(diffPathAbsolute, filePathBefore);
// TODO: add path to manifest JSON
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException e){
LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
// TODO: throw exception?
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) {
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
}
}
private static byte[] digestFromPath(Path path) {
try {
return Crypto.digest(Files.readAllBytes(path));
} catch (IOException e) {
return null;
}
}
private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
if (!Files.exists(source)) {
throw new IOException(String.format("File not found: %s", source.toString()));
}
Path dest = Paths.get(base.toString(), relativePath.toString());
LOGGER.trace("Copying {} to {}", source, dest);
Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
}
private static void markFilePathAsRemoved(Path base, Path relativePath) throws IOException {
String newFilename = relativePath.toString().concat(".removed");
Path dest = Paths.get(base.toString(), newFilename);
File file = new File(dest.toString());
File parent = file.getParentFile();
if (parent != null) {
parent.mkdirs();
}
LOGGER.info("Creating file {}", dest);
file.createNewFile();
}
public Path getDiffPath() {
return this.diffPath;
}
}

190
src/main/java/org/qortal/storage/DataFileMerge.java Normal file
View File

@ -0,0 +1,190 @@
package org.qortal.storage;
import org.apache.commons.io.FileUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.Crypto;
import org.qortal.utils.FilesystemUtils;
import java.io.File;
import java.io.IOException;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
public class DataFileMerge {
private static final Logger LOGGER = LogManager.getLogger(DataFileMerge.class);
private Path pathBefore;
private Path pathAfter;
private Path mergePath;
public DataFileMerge(Path pathBefore, Path pathAfter) {
this.pathBefore = pathBefore;
this.pathAfter = pathAfter;
}
public void compute() throws IOException {
try {
this.preExecute();
this.copyPreviousStateToMergePath();
this.findDifferences();
} finally {
this.postExecute();
}
}
private void preExecute() {
this.createOutputDirectory();
}
private void postExecute() {
}
private void createOutputDirectory() {
// Ensure temp folder exists
Path tempDir;
try {
tempDir = Files.createTempDirectory("qortal-diff");
} catch (IOException e) {
throw new IllegalStateException("Unable to create temp directory");
}
this.mergePath = tempDir;
}
private void copyPreviousStateToMergePath() throws IOException {
DataFileMerge.copyDirPathToBaseDir(this.pathBefore, this.mergePath, Paths.get(""));
}
private void findDifferences() {
final Path pathBeforeAbsolute = this.pathBefore.toAbsolutePath();
final Path pathAfterAbsolute = this.pathAfter.toAbsolutePath();
final Path mergePathAbsolute = this.mergePath.toAbsolutePath();
// LOGGER.info("this.pathBefore: {}", this.pathBefore);
// LOGGER.info("this.pathAfter: {}", this.pathAfter);
// LOGGER.info("pathBeforeAbsolute: {}", pathBeforeAbsolute);
// LOGGER.info("pathAfterAbsolute: {}", pathAfterAbsolute);
// LOGGER.info("mergePathAbsolute: {}", mergePathAbsolute);
try {
// Check for additions or modifications
Files.walkFileTree(this.pathAfter, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path after, BasicFileAttributes attrs) {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path after, BasicFileAttributes attrs) throws IOException {
Path filePathAfter = pathAfterAbsolute.relativize(after.toAbsolutePath());
Path filePathBefore = pathBeforeAbsolute.resolve(filePathAfter);
boolean wasAdded = false;
boolean wasModified = false;
boolean wasRemoved = false;
if (after.toString().endsWith(".removed")) {
LOGGER.trace("File was removed: {}", after.toString());
wasRemoved = true;
}
else if (!Files.exists(filePathBefore)) {
LOGGER.trace("File was added: {}", after.toString());
wasAdded = true;
}
else if (Files.size(after) != Files.size(filePathBefore)) {
// Check file size first because it's quicker
LOGGER.trace("File size was modified: {}", after.toString());
wasModified = true;
}
else if (!Arrays.equals(DataFileMerge.digestFromPath(after), DataFileMerge.digestFromPath(filePathBefore))) {
// Check hashes as a last resort
LOGGER.trace("File contents were modified: {}", after.toString());
wasModified = true;
}
if (wasAdded | wasModified) {
DataFileMerge.copyFilePathToBaseDir(after, mergePathAbsolute, filePathAfter);
}
if (wasRemoved) {
if (filePathAfter.toString().endsWith(".removed")) {
// Trim the ".removed"
Path filePathAfterTrimmed = Paths.get(filePathAfter.toString().substring(0, filePathAfter.toString().length()-8));
DataFileMerge.deletePathInBaseDir(mergePathAbsolute, filePathAfterTrimmed);
}
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException e){
LOGGER.info("File visit failed: {}, error: {}", file.toString(), e.getMessage());
// TODO: throw exception?
return FileVisitResult.TERMINATE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) {
return FileVisitResult.CONTINUE;
}
});
} catch (IOException e) {
LOGGER.info("IOException when walking through file tree: {}", e.getMessage());
}
}
private static byte[] digestFromPath(Path path) {
try {
return Crypto.digest(Files.readAllBytes(path));
} catch (IOException e) {
return null;
}
}
private static void copyFilePathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
if (!Files.exists(source)) {
throw new IOException(String.format("File not found: %s", source.toString()));
}
Path dest = Paths.get(base.toString(), relativePath.toString());
LOGGER.trace("Copying {} to {}", source, dest);
Files.copy(source, dest, StandardCopyOption.REPLACE_EXISTING);
}
private static void copyDirPathToBaseDir(Path source, Path base, Path relativePath) throws IOException {
if (!Files.exists(source)) {
throw new IOException(String.format("File not found: %s", source.toString()));
}
Path dest = Paths.get(base.toString(), relativePath.toString());
LOGGER.trace("Copying {} to {}", source, dest);
FilesystemUtils.copyDirectory(source.toString(), dest.toString());
}
private static void deletePathInBaseDir(Path base, Path relativePath) throws IOException {
Path dest = Paths.get(base.toString(), relativePath.toString());
File file = new File(dest.toString());
if (file.exists() && file.isFile()) {
LOGGER.trace("Deleting file {}", dest);
Files.delete(dest);
}
if (file.exists() && file.isDirectory()) {
LOGGER.trace("Deleting directory {}", dest);
FileUtils.deleteDirectory(file);
}
}
public Path getMergePath() {
return this.mergePath;
}
}

66
src/main/java/org/qortal/storage/DataFilePatches.java Normal file
View File

@ -0,0 +1,66 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.repository.DataException;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
public class DataFilePatches {
private static final Logger LOGGER = LogManager.getLogger(DataFilePatches.class);
private List<Path> paths;
private Path finalPath;
public DataFilePatches(List<Path> paths) {
this.paths = paths;
}
public void applyPatches() throws DataException, IOException {
try {
this.preExecute();
this.process();
} finally {
this.postExecute();
}
}
private void preExecute() {
if (this.paths == null || this.paths.isEmpty()) {
throw new IllegalStateException(String.format("No paths available to build latest state"));
}
}
private void postExecute() {
}
private void process() throws IOException {
if (this.paths.size() == 1) {
// No patching needed
this.finalPath = this.paths.get(0);
return;
}
Path pathBefore = this.paths.get(0);
// Loop from the second path onwards
for (int i=1; i<paths.size(); i++) {
Path pathAfter = this.paths.get(i);
DataFileCombiner combiner = new DataFileCombiner(pathBefore, pathAfter);
combiner.combine();
pathBefore = combiner.getFinalPath(); // TODO: cleanup
}
this.finalPath = pathBefore;
}
public Path getFinalPath() {
return this.finalPath;
}
}

169
src/main/java/org/qortal/storage/DataFileReader.java
View File

@ -1,13 +1,17 @@
package org.qortal.storage;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.crypto.AES;
import org.qortal.data.transaction.ArbitraryTransactionData;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
import org.qortal.storage.DataFile.*;
import org.qortal.transform.Transformer;
import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils;
import org.qortal.utils.ZipUtils;
import javax.crypto.BadPaddingException;
@ -17,9 +21,8 @@ import javax.crypto.SecretKey;
import javax.crypto.spec.SecretKeySpec;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.*;
import java.nio.file.attribute.BasicFileAttributes;
import java.security.InvalidAlgorithmParameterException;
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
@ -27,33 +30,38 @@ import java.util.Arrays;
public class DataFileReader {
private static final Logger LOGGER = LogManager.getLogger(DataFileReader.class);
private String resourceId;
private ResourceIdType resourceIdType;
private Service service;
private ArbitraryTransactionData transactionData;
private String secret58;
private Path filePath;
private DataFile dataFile;
// Intermediate paths
private Path workingPath;
private Path uncompressedPath;
private Path unencryptedPath;
public DataFileReader(String resourceId, ResourceIdType resourceIdType) {
public DataFileReader(String resourceId, ResourceIdType resourceIdType, Service service) {
this.resourceId = resourceId;
this.resourceIdType = resourceIdType;
this.service = service;
}
public void load(boolean overwrite) throws IllegalStateException, IOException, DataException {
try {
this.preExecute();
// Do nothing if files already exist and overwrite is set to false
if (Files.exists(this.uncompressedPath) && !overwrite) {
if (!overwrite && Files.exists(this.uncompressedPath)
&& !FilesystemUtils.isDirectoryEmpty(this.uncompressedPath)) {
this.filePath = this.uncompressedPath;
return;
}
this.deleteExistingFiles();
this.fetch();
this.decrypt();
this.uncompress();
@ -65,8 +73,7 @@ public class DataFileReader {
private void preExecute() {
this.createWorkingDirectory();
// Initialize unzipped path as it's used in a few places
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
this.createUncompressedDirectory();
}
private void postExecute() throws IOException {
@ -85,15 +92,70 @@ public class DataFileReader {
this.workingPath = tempDir;
}
private void createUncompressedDirectory() {
// Use the system tmpdir as our base, as it is deterministic
this.uncompressedPath = Paths.get(this.workingPath.toString() + File.separator + "data");
try {
Files.createDirectories(this.uncompressedPath);
} catch (IOException e) {
throw new IllegalStateException("Unable to create temp directory");
}
}
private void deleteExistingFiles() {
final Path uncompressedPath = this.uncompressedPath;
if (uncompressedPath != null) {
if (Files.exists(uncompressedPath)) {
LOGGER.trace("Attempting to delete path {}", this.uncompressedPath);
try {
Files.walkFileTree(uncompressedPath, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
Files.delete(file);
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException e) throws IOException {
// Don't delete the parent directory, as we want to leave an empty folder
if (dir.compareTo(uncompressedPath) == 0) {
return FileVisitResult.CONTINUE;
}
if (e == null) {
Files.delete(dir);
return FileVisitResult.CONTINUE;
} else {
throw e;
}
}
});
} catch (IOException e) {
LOGGER.info("Unable to delete file or directory: {}", e.getMessage());
}
}
}
}
private void fetch() throws IllegalStateException, IOException, DataException {
switch (resourceIdType) {
case FILE_HASH:
this.fetchFromFileHash();
break;
case NAME:
this.fetchFromName();
break;
case SIGNATURE:
this.fetchFromSignature();
break;
case FILE_HASH:
this.fetchFromFileHash();
case TRANSACTION_DATA:
this.fetchFromTransactionData(this.transactionData);
break;
default:
@ -101,9 +163,30 @@ public class DataFileReader {
}
}
private void fetchFromFileHash() {
// Load data file directly from the hash
DataFile dataFile = DataFile.fromHash58(resourceId);
// Set filePath to the location of the DataFile
this.filePath = Paths.get(dataFile.getFilePath());
}
private void fetchFromName() throws IllegalStateException, IOException, DataException {
// Build the existing state using past transactions
DataFileBuilder builder = new DataFileBuilder(this.resourceId, this.service);
builder.build();
Path builtPath = builder.getFinalPath();
if (builtPath == null) {
throw new IllegalStateException("Unable to build path");
}
// Set filePath to the builtPath
this.filePath = builtPath;
}
private void fetchFromSignature() throws IllegalStateException, IOException, DataException {
// Load the full transaction data so we can access the file hashes
// Load the full transaction data from the database so we can access the file hashes
ArbitraryTransactionData transactionData;
try (final Repository repository = RepositoryManager.getRepository()) {
transactionData = (ArbitraryTransactionData) repository.getTransactionRepository().fromSignature(Base58.decode(resourceId));
@ -112,6 +195,14 @@ public class DataFileReader {
throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId));
}
this.fetchFromTransactionData(transactionData);
}
private void fetchFromTransactionData(ArbitraryTransactionData transactionData) throws IllegalStateException, IOException, DataException {
if (!(transactionData instanceof ArbitraryTransactionData)) {
throw new IllegalStateException(String.format("Transaction data not found for signature %s", this.resourceId));
}
// Load hashes
byte[] digest = transactionData.getData();
byte[] chunkHashes = transactionData.getChunkHashes();
@ -123,19 +214,19 @@ public class DataFileReader {
}
// Load data file(s)
this.dataFile = DataFile.fromHash(digest);
if (!this.dataFile.exists()) {
if (!this.dataFile.allChunksExist(chunkHashes)) {
DataFile dataFile = DataFile.fromHash(digest);
if (!dataFile.exists()) {
if (!dataFile.allChunksExist(chunkHashes)) {
// TODO: fetch them?
throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile));
}
// We have all the chunks but not the complete file, so join them
this.dataFile.addChunkHashes(chunkHashes);
this.dataFile.join();
dataFile.addChunkHashes(chunkHashes);
dataFile.join();
}
// If the complete file still doesn't exist then something went wrong
if (!this.dataFile.exists()) {
if (!dataFile.exists()) {
throw new IOException(String.format("File doesn't exist: %s", dataFile));
}
// Ensure the complete hash matches the joined chunks
@ -146,13 +237,6 @@ public class DataFileReader {
this.filePath = Paths.get(dataFile.getFilePath());
}
private void fetchFromFileHash() {
// Load data file directly from the hash
this.dataFile = DataFile.fromHash58(resourceId);
// Set filePath to the location of the DataFile
this.filePath = Paths.get(dataFile.getFilePath());
}
private void decrypt() {
// Decrypt if we have the secret key.
byte[] secret = this.secret58 != null ? Base58.decode(this.secret58) : null;
@ -168,15 +252,26 @@ public class DataFileReader {
} catch (NoSuchAlgorithmException | InvalidAlgorithmParameterException | NoSuchPaddingException
| BadPaddingException | IllegalBlockSizeException | IOException | InvalidKeyException e) {
throw new IllegalStateException(String.format("Unable to decrypt file %s: %s", dataFile, e.getMessage()));
throw new IllegalStateException(String.format("Unable to decrypt file at path %s: %s", this.filePath, e.getMessage()));
}
} else {
// Assume it is unencrypted. We may block this in the future.
this.filePath = Paths.get(this.dataFile.getFilePath());
// Assume it is unencrypted. This will be the case when we have built a custom path by combining
// multiple decrypted archives into a single state.
}
}
private void uncompress() throws IOException {
if (this.filePath == null || !Files.exists(this.filePath)) {
throw new IllegalStateException("Can't uncompress non-existent file path");
}
File file = new File(this.filePath.toString());
if (file.isDirectory()) {
// Already a directory - nothing to uncompress
// We still need to copy the directory to its final destination if it's not already there
this.copyFilePathToFinalDestination();
return;
}
try {
// TODO: compression types
//if (transactionData.getCompression() == ArbitraryTransactionData.Compression.ZIP) {
@ -191,6 +286,20 @@ public class DataFileReader {
this.filePath = this.uncompressedPath;
}
private void copyFilePathToFinalDestination() throws IOException {
if (this.filePath.compareTo(this.uncompressedPath) != 0) {
File source = new File(this.filePath.toString());
File dest = new File(this.uncompressedPath.toString());
if (source == null || !source.exists()) {
throw new IllegalStateException("Source directory doesn't exist");
}
if (dest == null || !dest.exists()) {
throw new IllegalStateException("Destination directory doesn't exist");
}
FilesystemUtils.copyDirectory(source.toString(), dest.toString());
}
}
private void cleanupFilesystem() throws IOException {
// Clean up
if (this.uncompressedPath != null) {
@ -202,6 +311,10 @@ public class DataFileReader {
}
public void setTransactionData(ArbitraryTransactionData transactionData) {
this.transactionData = transactionData;
}
public void setSecret58(String secret58) {
this.secret58 = secret58;
}

40
src/main/java/org/qortal/storage/DataFileWriter.java
View File

@ -5,6 +5,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.crypto.AES;
import org.qortal.repository.DataException;
import org.qortal.storage.DataFile.*;
import org.qortal.utils.ZipUtils;
@ -26,6 +27,8 @@ public class DataFileWriter {
private static final Logger LOGGER = LogManager.getLogger(DataFileWriter.class);
private Path filePath;
private String name;
private Service service;
private Method method;
private Compression compression;
@ -37,15 +40,18 @@ public class DataFileWriter {
private Path compressedPath;
private Path encryptedPath;
public DataFileWriter(Path filePath, Method method, Compression compression) {
public DataFileWriter(Path filePath, String name, Service service, Method method, Compression compression) {
this.filePath = filePath;
this.name = name;
this.service = service;
this.method = method;
this.compression = compression;
}
public void save() throws IllegalStateException, IOException {
public void save() throws IllegalStateException, IOException, DataException {
try {
this.preExecute();
this.process();
this.compress();
this.encrypt();
this.split();
@ -82,6 +88,36 @@ public class DataFileWriter {
this.workingPath = tempDir;
}
private void process() throws DataException, IOException {
switch (this.method) {
case PUT:
// Nothing to do
break;
case PATCH:
this.processPatch();
break;
default:
throw new IllegalStateException(String.format("Unknown method specified: %s", method.toString()));
}
}
private void processPatch() throws DataException, IOException {
// Build the existing state using past transactions
DataFileBuilder builder = new DataFileBuilder(this.name, this.service);
builder.build();
Path builtPath = builder.getFinalPath();
// Compute a diff of the latest changes on top of the previous state
// Then use only the differences as our data payload
DataFileCreatePatch patch = new DataFileCreatePatch(builtPath, this.filePath);
patch.create();
this.filePath = patch.getFinalPath();
}
private void compress() {
// Compress the data if requested
if (this.compression != Compression.NONE) {

3
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -108,7 +108,8 @@ public class ArbitraryTransaction extends Transaction {
if (chunkHashes == null && expectedChunkHashesSize > 0) {
return ValidationResult.INVALID_DATA_LENGTH;
}
if (chunkHashes.length != expectedChunkHashesSize) {
int chunkHashesLength = chunkHashes != null ? chunkHashes.length : 0;
if (chunkHashesLength != expectedChunkHashesSize) {
return ValidationResult.INVALID_DATA_LENGTH;
}
}

34
src/main/java/org/qortal/utils/FilesystemUtils.java Normal file
View File

@ -0,0 +1,34 @@
package org.qortal.utils;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
public class FilesystemUtils {
public static boolean isDirectoryEmpty(Path path) throws IOException {
if (Files.isDirectory(path)) {
try (DirectoryStream<Path> directory = Files.newDirectoryStream(path)) {
return !directory.iterator().hasNext();
}
}
return false;
}
public static void copyDirectory(String sourceDirectoryLocation, String destinationDirectoryLocation) throws IOException {
Files.walk(Paths.get(sourceDirectoryLocation))
.forEach(source -> {
Path destination = Paths.get(destinationDirectoryLocation, source.toString()
.substring(sourceDirectoryLocation.length()));
try {
Files.copy(source, destination);
} catch (IOException e) {
e.printStackTrace();
}
});
}
}