forked from Qortal/qortal
Renamed DataFile to ArbitraryDataFile, and DataFileChunk to ArbitraryDataFileChunk
This commit is contained in:
CalDescent
parent
16d93b1775
commit
da6b341b63
@ -45,8 +45,8 @@ import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.DataFileChunk;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.storage.ArbitraryDataFileChunk;
|
||||
import org.qortal.storage.ArbitraryDataWriter;
|
||||
import org.qortal.transaction.ArbitraryTransaction;
|
||||
import org.qortal.transaction.Transaction;
|
||||
@ -281,28 +281,28 @@ public class ArbitraryResource {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
|
||||
DataFile dataFile = arbitraryDataWriter.getDataFile();
|
||||
if (dataFile == null) {
|
||||
ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
|
||||
if (arbitraryDataFile == null) {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
|
||||
DataFile.ValidationResult validationResult = dataFile.isValid();
|
||||
if (validationResult != DataFile.ValidationResult.OK) {
|
||||
ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid();
|
||||
if (validationResult != ArbitraryDataFile.ValidationResult.OK) {
|
||||
LOGGER.error("Invalid file: {}", validationResult);
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
LOGGER.info("Whole file digest: {}", dataFile.digest58());
|
||||
LOGGER.info("Whole file digest: {}", arbitraryDataFile.digest58());
|
||||
|
||||
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE);
|
||||
int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
|
||||
if (chunkCount == 0) {
|
||||
LOGGER.error("No chunks created");
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
|
||||
|
||||
String digest58 = dataFile.digest58();
|
||||
String digest58 = arbitraryDataFile.digest58();
|
||||
if (digest58 == null) {
|
||||
LOGGER.error("Unable to calculate digest");
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
@ -313,12 +313,12 @@ public class ArbitraryResource {
|
||||
|
||||
final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP,
|
||||
lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null);
|
||||
final int size = (int)dataFile.size();
|
||||
final int size = (int) arbitraryDataFile.size();
|
||||
final int version = 5;
|
||||
final int nonce = 0;
|
||||
final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
|
||||
final byte[] digest = dataFile.digest();
|
||||
final byte[] chunkHashes = dataFile.chunkHashes();
|
||||
final byte[] digest = arbitraryDataFile.digest();
|
||||
final byte[] chunkHashes = arbitraryDataFile.chunkHashes();
|
||||
final List<PaymentData> payments = new ArrayList<>();
|
||||
|
||||
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
|
||||
@ -330,7 +330,7 @@ public class ArbitraryResource {
|
||||
|
||||
Transaction.ValidationResult result = transaction.isValidUnconfirmed();
|
||||
if (result != Transaction.ValidationResult.OK) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw TransactionsResource.createTransactionInvalidException(request, result);
|
||||
}
|
||||
|
||||
@ -338,14 +338,14 @@ public class ArbitraryResource {
|
||||
return Base58.encode(bytes);
|
||||
|
||||
} catch (DataException e) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
LOGGER.error("Repository issue when uploading data", e);
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
|
||||
} catch (TransformationException e) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
|
||||
} catch (IllegalStateException e) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
LOGGER.error("Invalid upload data", e);
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
|
||||
}
|
||||
@ -380,8 +380,8 @@ public class ArbitraryResource {
|
||||
public String deleteFile(String hash58) {
|
||||
Security.checkApiCallAllowed(request);
|
||||
|
||||
DataFile dataFile = DataFile.fromHash58(hash58);
|
||||
if (dataFile.delete()) {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58);
|
||||
if (arbitraryDataFile.delete()) {
|
||||
return "true";
|
||||
}
|
||||
return "false";
|
||||
@ -503,9 +503,9 @@ public class ArbitraryResource {
|
||||
private boolean requestFile(String hash58, Peer targetPeer) {
|
||||
try (final Repository repository = RepositoryManager.getRepository()) {
|
||||
|
||||
DataFile dataFile = DataFile.fromHash58(hash58);
|
||||
if (dataFile.exists()) {
|
||||
LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58);
|
||||
if (arbitraryDataFile.exists()) {
|
||||
LOGGER.info("Data file {} already exists but we'll request it anyway", arbitraryDataFile);
|
||||
}
|
||||
|
||||
byte[] digest = null;
|
||||
@ -526,11 +526,11 @@ public class ArbitraryResource {
|
||||
}
|
||||
|
||||
DataFileMessage dataFileMessage = (DataFileMessage) message;
|
||||
dataFile = dataFileMessage.getDataFile();
|
||||
if (dataFile == null || !dataFile.exists()) {
|
||||
arbitraryDataFile = dataFileMessage.getArbitraryDataFile();
|
||||
if (arbitraryDataFile == null || !arbitraryDataFile.exists()) {
|
||||
return false;
|
||||
}
|
||||
LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size()));
|
||||
LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getArbitraryDataFile(), dataFileMessage.getArbitraryDataFile().size()));
|
||||
return true;
|
||||
} catch (ApiException e) {
|
||||
throw e;
|
||||
@ -571,22 +571,22 @@ public class ArbitraryResource {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
|
||||
}
|
||||
|
||||
DataFile dataFile = DataFile.fromHash58(combinedHash);
|
||||
if (dataFile.exists()) {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(combinedHash);
|
||||
if (arbitraryDataFile.exists()) {
|
||||
LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash);
|
||||
}
|
||||
|
||||
String hash58List[] = files.split(",");
|
||||
for (String hash58 : hash58List) {
|
||||
if (hash58 != null) {
|
||||
DataFileChunk chunk = DataFileChunk.fromHash58(hash58);
|
||||
dataFile.addChunk(chunk);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash58(hash58);
|
||||
arbitraryDataFile.addChunk(chunk);
|
||||
}
|
||||
}
|
||||
boolean success = dataFile.join();
|
||||
boolean success = arbitraryDataFile.join();
|
||||
if (success) {
|
||||
if (combinedHash.equals(dataFile.digest58())) {
|
||||
LOGGER.info("Valid hash {} after joining {} files", dataFile.digest58(), dataFile.chunkCount());
|
||||
if (combinedHash.equals(arbitraryDataFile.digest58())) {
|
||||
LOGGER.info("Valid hash {} after joining {} files", arbitraryDataFile.digest58(), arbitraryDataFile.chunkCount());
|
||||
return Response.ok("true").build();
|
||||
}
|
||||
}
|
||||
|
||||
@ -38,8 +38,8 @@ import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.settings.Settings;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.DataFile.*;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.storage.ArbitraryDataFile.*;
|
||||
import org.qortal.storage.ArbitraryDataReader;
|
||||
import org.qortal.storage.ArbitraryDataWriter;
|
||||
import org.qortal.transaction.ArbitraryTransaction;
|
||||
@ -112,12 +112,12 @@ public class WebsiteResource {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
|
||||
DataFile dataFile = arbitraryDataWriter.getDataFile();
|
||||
if (dataFile == null) {
|
||||
ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
|
||||
if (arbitraryDataFile == null) {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
|
||||
String digest58 = dataFile.digest58();
|
||||
String digest58 = arbitraryDataFile.digest58();
|
||||
if (digest58 == null) {
|
||||
LOGGER.error("Unable to calculate digest");
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
@ -130,13 +130,13 @@ public class WebsiteResource {
|
||||
|
||||
final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP,
|
||||
lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null);
|
||||
final int size = (int)dataFile.size();
|
||||
final int size = (int) arbitraryDataFile.size();
|
||||
final int version = 5;
|
||||
final int nonce = 0;
|
||||
byte[] secret = dataFile.getSecret();
|
||||
byte[] secret = arbitraryDataFile.getSecret();
|
||||
final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
|
||||
final byte[] digest = dataFile.digest();
|
||||
final byte[] chunkHashes = dataFile.chunkHashes();
|
||||
final byte[] digest = arbitraryDataFile.digest();
|
||||
final byte[] chunkHashes = arbitraryDataFile.chunkHashes();
|
||||
final List<PaymentData> payments = new ArrayList<>();
|
||||
|
||||
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
|
||||
@ -149,7 +149,7 @@ public class WebsiteResource {
|
||||
|
||||
Transaction.ValidationResult result = transaction.isValidUnconfirmed();
|
||||
if (result != Transaction.ValidationResult.OK) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw TransactionsResource.createTransactionInvalidException(request, result);
|
||||
}
|
||||
|
||||
@ -157,10 +157,10 @@ public class WebsiteResource {
|
||||
return Base58.encode(bytes);
|
||||
|
||||
} catch (TransformationException e) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
|
||||
} catch (DataException e) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
|
||||
}
|
||||
}
|
||||
@ -212,11 +212,11 @@ public class WebsiteResource {
|
||||
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
|
||||
}
|
||||
|
||||
DataFile dataFile = arbitraryDataWriter.getDataFile();
|
||||
if (dataFile != null) {
|
||||
String digest58 = dataFile.digest58();
|
||||
ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
|
||||
if (arbitraryDataFile != null) {
|
||||
String digest58 = arbitraryDataFile.digest58();
|
||||
if (digest58 != null) {
|
||||
return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(dataFile.getSecret());
|
||||
return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret());
|
||||
}
|
||||
}
|
||||
return "Unable to generate preview URL";
|
||||
|
||||
@ -13,8 +13,8 @@ import org.qortal.network.message.*;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.DataFileChunk;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.storage.ArbitraryDataFileChunk;
|
||||
import org.qortal.transaction.ArbitraryTransaction;
|
||||
import org.qortal.transaction.Transaction.TransactionType;
|
||||
import org.qortal.utils.Base58;
|
||||
@ -166,7 +166,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
return true;
|
||||
}
|
||||
|
||||
private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException {
|
||||
private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException {
|
||||
String hash58 = Base58.encode(hash);
|
||||
LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer));
|
||||
arbitraryDataFileRequests.put(hash58, NTP.getTime());
|
||||
@ -181,7 +181,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
}
|
||||
|
||||
DataFileMessage dataFileMessage = (DataFileMessage) message;
|
||||
return dataFileMessage.getDataFile();
|
||||
return dataFileMessage.getArbitraryDataFile();
|
||||
}
|
||||
|
||||
public void cleanupRequestCache(long now) {
|
||||
@ -269,13 +269,13 @@ public class ArbitraryDataManager extends Thread {
|
||||
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
|
||||
|
||||
// Load data file(s)
|
||||
DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData());
|
||||
dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData());
|
||||
arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
|
||||
|
||||
// Check all hashes exist
|
||||
for (byte[] hash : hashes) {
|
||||
//LOGGER.info("Received hash {}", Base58.encode(hash));
|
||||
if (!dataFile.containsChunk(hash)) {
|
||||
if (!arbitraryDataFile.containsChunk(hash)) {
|
||||
LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58);
|
||||
return;
|
||||
}
|
||||
@ -287,14 +287,14 @@ public class ArbitraryDataManager extends Thread {
|
||||
|
||||
// Now fetch actual data from this peer
|
||||
for (byte[] hash : hashes) {
|
||||
if (!dataFile.chunkExists(hash)) {
|
||||
if (!arbitraryDataFile.chunkExists(hash)) {
|
||||
// Only request the file if we aren't already requesting it from someone else
|
||||
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
|
||||
DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash);
|
||||
LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer);
|
||||
ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash);
|
||||
LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer);
|
||||
}
|
||||
else {
|
||||
LOGGER.info("Already requesting data file {}", dataFile);
|
||||
LOGGER.info("Already requesting data file {}", arbitraryDataFile);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -318,15 +318,15 @@ public class ArbitraryDataManager extends Thread {
|
||||
byte[] hash = getDataFileMessage.getHash();
|
||||
Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet();
|
||||
|
||||
DataFile dataFile = DataFile.fromHash(hash);
|
||||
if (dataFile.exists()) {
|
||||
DataFileMessage dataFileMessage = new DataFileMessage(dataFile);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
|
||||
if (arbitraryDataFile.exists()) {
|
||||
DataFileMessage dataFileMessage = new DataFileMessage(arbitraryDataFile);
|
||||
dataFileMessage.setId(message.getId());
|
||||
if (!peer.sendMessage(dataFileMessage)) {
|
||||
LOGGER.info("Couldn't sent file");
|
||||
peer.disconnect("failed to send file");
|
||||
}
|
||||
LOGGER.info("Sent file {}", dataFile);
|
||||
LOGGER.info("Sent file {}", arbitraryDataFile);
|
||||
}
|
||||
else {
|
||||
|
||||
@ -334,7 +334,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement();
|
||||
|
||||
// Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
|
||||
LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile));
|
||||
LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
|
||||
|
||||
// We'll send empty block summaries message as it's very short
|
||||
// TODO: use a different message type here
|
||||
@ -344,7 +344,7 @@ public class ArbitraryDataManager extends Thread {
|
||||
LOGGER.info("Couldn't sent file-unknown response");
|
||||
peer.disconnect("failed to send file-unknown response");
|
||||
}
|
||||
LOGGER.info("Sent file-unknown response for file {}", dataFile);
|
||||
LOGGER.info("Sent file-unknown response for file {}", arbitraryDataFile);
|
||||
}
|
||||
}
|
||||
|
||||
@ -367,10 +367,10 @@ public class ArbitraryDataManager extends Thread {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load file(s) and add any that exist to the list of hashes
|
||||
DataFile dataFile = DataFile.fromHash(hash);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
dataFile.addChunkHashes(chunkHashes);
|
||||
for (DataFileChunk dataFileChunk : dataFile.getChunks()) {
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
for (ArbitraryDataFileChunk dataFileChunk : arbitraryDataFile.getChunks()) {
|
||||
if (dataFileChunk.exists()) {
|
||||
hashes.add(dataFileChunk.getHash());
|
||||
//LOGGER.info("Added hash {}", dataFileChunk.getHash58());
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
package org.qortal.network.message;
|
||||
|
||||
import com.google.common.primitives.Ints;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.io.IOException;
|
||||
@ -10,22 +10,22 @@ import java.nio.ByteBuffer;
|
||||
|
||||
public class DataFileMessage extends Message {
|
||||
|
||||
private final DataFile dataFile;
|
||||
private final ArbitraryDataFile arbitraryDataFile;
|
||||
|
||||
public DataFileMessage(DataFile dataFile) {
|
||||
public DataFileMessage(ArbitraryDataFile arbitraryDataFile) {
|
||||
super(MessageType.DATA_FILE);
|
||||
|
||||
this.dataFile = dataFile;
|
||||
this.arbitraryDataFile = arbitraryDataFile;
|
||||
}
|
||||
|
||||
public DataFileMessage(int id, DataFile dataFile) {
|
||||
public DataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) {
|
||||
super(id, MessageType.DATA_FILE);
|
||||
|
||||
this.dataFile = dataFile;
|
||||
this.arbitraryDataFile = arbitraryDataFile;
|
||||
}
|
||||
|
||||
public DataFile getDataFile() {
|
||||
return this.dataFile;
|
||||
public ArbitraryDataFile getArbitraryDataFile() {
|
||||
return this.arbitraryDataFile;
|
||||
}
|
||||
|
||||
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
|
||||
@ -36,18 +36,18 @@ public class DataFileMessage extends Message {
|
||||
|
||||
byte[] data = new byte[dataLength];
|
||||
byteBuffer.get(data);
|
||||
DataFile dataFile = new DataFile(data);
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data);
|
||||
|
||||
return new DataFileMessage(id, dataFile);
|
||||
return new DataFileMessage(id, arbitraryDataFile);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected byte[] toData() {
|
||||
if (this.dataFile == null) {
|
||||
if (this.arbitraryDataFile == null) {
|
||||
return null;
|
||||
}
|
||||
|
||||
byte[] data = this.dataFile.getBytes();
|
||||
byte[] data = this.arbitraryDataFile.getBytes();
|
||||
if (data == null) {
|
||||
return null;
|
||||
}
|
||||
@ -66,7 +66,7 @@ public class DataFileMessage extends Message {
|
||||
}
|
||||
|
||||
public DataFileMessage cloneWithNewId(int newId) {
|
||||
DataFileMessage clone = new DataFileMessage(this.dataFile);
|
||||
DataFileMessage clone = new DataFileMessage(this.arbitraryDataFile);
|
||||
clone.setId(newId);
|
||||
return clone;
|
||||
}
|
||||
|
||||
@ -10,7 +10,7 @@ import org.qortal.data.transaction.BaseTransactionData;
|
||||
import org.qortal.data.transaction.TransactionData;
|
||||
import org.qortal.repository.ArbitraryRepository;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.transaction.Transaction.ApprovalStatus;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
@ -55,18 +55,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
DataFile dataFile = DataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
dataFile.addChunkHashes(chunkHashes);
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
|
||||
// Check if we already have the complete data file
|
||||
if (dataFile.exists()) {
|
||||
if (arbitraryDataFile.exists()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Alternatively, if we have all the chunks, then it's safe to assume the data is local
|
||||
if (dataFile.allChunksExist(chunkHashes)) {
|
||||
if (arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@ -90,23 +90,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = transactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
DataFile dataFile = DataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
dataFile.addChunkHashes(chunkHashes);
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
|
||||
// If we have the complete data file, return it
|
||||
if (dataFile.exists()) {
|
||||
return dataFile.getBytes();
|
||||
if (arbitraryDataFile.exists()) {
|
||||
return arbitraryDataFile.getBytes();
|
||||
}
|
||||
|
||||
// Alternatively, if we have all the chunks, combine them into a single file
|
||||
if (dataFile.allChunksExist(chunkHashes)) {
|
||||
dataFile.join();
|
||||
if (arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
arbitraryDataFile.join();
|
||||
|
||||
// Verify that the combined hash matches the expected hash
|
||||
if (digest.equals(dataFile.digest())) {
|
||||
return dataFile.getBytes();
|
||||
if (digest.equals(arbitraryDataFile.digest())) {
|
||||
return arbitraryDataFile.getBytes();
|
||||
}
|
||||
}
|
||||
|
||||
@ -134,29 +134,29 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
arbitraryTransactionData.setDataType(DataType.DATA_HASH);
|
||||
|
||||
// Create DataFile
|
||||
DataFile dataFile = new DataFile(rawData);
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(rawData);
|
||||
|
||||
// Verify that the data file is valid, and that it matches the expected hash
|
||||
DataFile.ValidationResult validationResult = dataFile.isValid();
|
||||
if (validationResult != DataFile.ValidationResult.OK) {
|
||||
dataFile.deleteAll();
|
||||
ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid();
|
||||
if (validationResult != ArbitraryDataFile.ValidationResult.OK) {
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw new DataException("Invalid data file when attempting to store arbitrary transaction data");
|
||||
}
|
||||
if (!dataHash.equals(dataFile.digest())) {
|
||||
dataFile.deleteAll();
|
||||
if (!dataHash.equals(arbitraryDataFile.digest())) {
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw new DataException("Could not verify hash when attempting to store arbitrary transaction data");
|
||||
}
|
||||
|
||||
// Now create chunks if needed
|
||||
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE);
|
||||
int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
|
||||
if (chunkCount > 0) {
|
||||
LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s")));
|
||||
LOGGER.info("{}", dataFile.printChunks());
|
||||
LOGGER.info("{}", arbitraryDataFile.printChunks());
|
||||
|
||||
// Verify that the chunk hashes match those in the transaction
|
||||
byte[] chunkHashes = dataFile.chunkHashes();
|
||||
byte[] chunkHashes = arbitraryDataFile.chunkHashes();
|
||||
if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) {
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data");
|
||||
}
|
||||
|
||||
@ -175,13 +175,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
|
||||
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
|
||||
|
||||
// Load data file(s)
|
||||
DataFile dataFile = DataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
if (chunkHashes != null && chunkHashes.length > 0) {
|
||||
dataFile.addChunkHashes(chunkHashes);
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
}
|
||||
|
||||
// Delete file and chunks
|
||||
dataFile.deleteAll();
|
||||
arbitraryDataFile.deleteAll();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
||||
@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.Service;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.storage.DataFile.ResourceIdType;
|
||||
import org.qortal.storage.ArbitraryDataFile.ResourceIdType;
|
||||
import org.qortal.utils.Base58;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
@ -20,7 +20,7 @@ import static java.util.Arrays.stream;
|
||||
import static java.util.stream.Collectors.toMap;
|
||||
|
||||
|
||||
public class DataFile {
|
||||
public class ArbitraryDataFile {
|
||||
|
||||
// Validation results
|
||||
public enum ValidationResult {
|
||||
@ -30,13 +30,13 @@ public class DataFile {
|
||||
|
||||
public final int value;
|
||||
|
||||
private static final Map<Integer, DataFile.ValidationResult> map = stream(DataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
|
||||
private static final Map<Integer, ArbitraryDataFile.ValidationResult> map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
|
||||
|
||||
ValidationResult(int value) {
|
||||
this.value = value;
|
||||
}
|
||||
|
||||
public static DataFile.ValidationResult valueOf(int value) {
|
||||
public static ArbitraryDataFile.ValidationResult valueOf(int value) {
|
||||
return map.get(value);
|
||||
}
|
||||
}
|
||||
@ -49,7 +49,7 @@ public class DataFile {
|
||||
NAME
|
||||
};
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(DataFile.class);
|
||||
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
|
||||
|
||||
public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
|
||||
public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
|
||||
@ -57,20 +57,20 @@ public class DataFile {
|
||||
|
||||
protected String filePath;
|
||||
protected String hash58;
|
||||
private ArrayList<DataFileChunk> chunks;
|
||||
private ArrayList<ArbitraryDataFileChunk> chunks;
|
||||
private byte[] secret;
|
||||
|
||||
public DataFile() {
|
||||
public ArbitraryDataFile() {
|
||||
}
|
||||
|
||||
public DataFile(String hash58) {
|
||||
public ArbitraryDataFile(String hash58) {
|
||||
this.createDataDirectory();
|
||||
this.filePath = DataFile.getOutputFilePath(hash58, false);
|
||||
this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false);
|
||||
this.chunks = new ArrayList<>();
|
||||
this.hash58 = hash58;
|
||||
}
|
||||
|
||||
public DataFile(byte[] fileContent) {
|
||||
public ArbitraryDataFile(byte[] fileContent) {
|
||||
if (fileContent == null) {
|
||||
LOGGER.error("fileContent is null");
|
||||
return;
|
||||
@ -95,28 +95,28 @@ public class DataFile {
|
||||
}
|
||||
}
|
||||
|
||||
public static DataFile fromHash58(String hash58) {
|
||||
return new DataFile(hash58);
|
||||
public static ArbitraryDataFile fromHash58(String hash58) {
|
||||
return new ArbitraryDataFile(hash58);
|
||||
}
|
||||
|
||||
public static DataFile fromHash(byte[] hash) {
|
||||
return DataFile.fromHash58(Base58.encode(hash));
|
||||
public static ArbitraryDataFile fromHash(byte[] hash) {
|
||||
return ArbitraryDataFile.fromHash58(Base58.encode(hash));
|
||||
}
|
||||
|
||||
public static DataFile fromPath(String path) {
|
||||
public static ArbitraryDataFile fromPath(String path) {
|
||||
File file = new File(path);
|
||||
if (file.exists()) {
|
||||
try {
|
||||
byte[] fileContent = Files.readAllBytes(file.toPath());
|
||||
byte[] digest = Crypto.digest(fileContent);
|
||||
DataFile dataFile = DataFile.fromHash(digest);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
|
||||
// Copy file to base directory if needed
|
||||
Path filePath = Paths.get(path);
|
||||
if (Files.exists(filePath) && !dataFile.isInBaseDirectory(path)) {
|
||||
dataFile.copyToDataDirectory(filePath);
|
||||
if (Files.exists(filePath) && !arbitraryDataFile.isInBaseDirectory(path)) {
|
||||
arbitraryDataFile.copyToDataDirectory(filePath);
|
||||
}
|
||||
return dataFile;
|
||||
return arbitraryDataFile;
|
||||
|
||||
} catch (IOException e) {
|
||||
LOGGER.error("Couldn't compute digest for DataFile");
|
||||
@ -125,8 +125,8 @@ public class DataFile {
|
||||
return null;
|
||||
}
|
||||
|
||||
public static DataFile fromFile(File file) {
|
||||
return DataFile.fromPath(file.getPath());
|
||||
public static ArbitraryDataFile fromFile(File file) {
|
||||
return ArbitraryDataFile.fromPath(file.getPath());
|
||||
}
|
||||
|
||||
private boolean createDataDirectory() {
|
||||
@ -188,7 +188,7 @@ public class DataFile {
|
||||
long fileSize = Files.size(path);
|
||||
if (fileSize > MAX_FILE_SIZE) {
|
||||
LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE));
|
||||
return DataFile.ValidationResult.FILE_TOO_LARGE;
|
||||
return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE;
|
||||
}
|
||||
|
||||
} catch (IOException e) {
|
||||
@ -198,7 +198,7 @@ public class DataFile {
|
||||
return ValidationResult.OK;
|
||||
}
|
||||
|
||||
public void addChunk(DataFileChunk chunk) {
|
||||
public void addChunk(ArbitraryDataFileChunk chunk) {
|
||||
this.chunks.add(chunk);
|
||||
}
|
||||
|
||||
@ -210,7 +210,7 @@ public class DataFile {
|
||||
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
|
||||
byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH];
|
||||
byteBuffer.get(chunkDigest);
|
||||
DataFileChunk chunk = DataFileChunk.fromHash(chunkDigest);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest);
|
||||
this.addChunk(chunk);
|
||||
}
|
||||
}
|
||||
@ -232,7 +232,7 @@ public class DataFile {
|
||||
out.write(buffer, 0, numberOfBytes);
|
||||
out.flush();
|
||||
|
||||
DataFileChunk chunk = new DataFileChunk(out.toByteArray());
|
||||
ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray());
|
||||
ValidationResult validationResult = chunk.isValid();
|
||||
if (validationResult == ValidationResult.OK) {
|
||||
this.chunks.add(chunk);
|
||||
@ -266,7 +266,7 @@ public class DataFile {
|
||||
// Join the chunks
|
||||
File outputFile = new File(this.filePath);
|
||||
try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
|
||||
for (DataFileChunk chunk : this.chunks) {
|
||||
for (ArbitraryDataFileChunk chunk : this.chunks) {
|
||||
File sourceFile = new File(chunk.filePath);
|
||||
BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile));
|
||||
byte[] buffer = new byte[2048];
|
||||
@ -323,7 +323,7 @@ public class DataFile {
|
||||
if (this.chunks != null && this.chunks.size() > 0) {
|
||||
Iterator iterator = this.chunks.iterator();
|
||||
while (iterator.hasNext()) {
|
||||
DataFileChunk chunk = (DataFileChunk) iterator.next();
|
||||
ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next();
|
||||
chunk.delete();
|
||||
iterator.remove();
|
||||
success = true;
|
||||
@ -376,7 +376,7 @@ public class DataFile {
|
||||
}
|
||||
|
||||
public boolean chunkExists(byte[] hash) {
|
||||
for (DataFileChunk chunk : this.chunks) {
|
||||
for (ArbitraryDataFileChunk chunk : this.chunks) {
|
||||
if (Arrays.equals(hash, chunk.getHash())) {
|
||||
return chunk.exists();
|
||||
}
|
||||
@ -389,7 +389,7 @@ public class DataFile {
|
||||
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
|
||||
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
|
||||
byteBuffer.get(chunkHash);
|
||||
DataFileChunk chunk = DataFileChunk.fromHash(chunkHash);
|
||||
ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash);
|
||||
if (!chunk.exists()) {
|
||||
return false;
|
||||
}
|
||||
@ -398,7 +398,7 @@ public class DataFile {
|
||||
}
|
||||
|
||||
public boolean containsChunk(byte[] hash) {
|
||||
for (DataFileChunk chunk : this.chunks) {
|
||||
for (ArbitraryDataFileChunk chunk : this.chunks) {
|
||||
if (Arrays.equals(hash, chunk.getHash())) {
|
||||
return true;
|
||||
}
|
||||
@ -419,7 +419,7 @@ public class DataFile {
|
||||
return this.chunks.size();
|
||||
}
|
||||
|
||||
public List<DataFileChunk> getChunks() {
|
||||
public List<ArbitraryDataFileChunk> getChunks() {
|
||||
return this.chunks;
|
||||
}
|
||||
|
||||
@ -432,7 +432,7 @@ public class DataFile {
|
||||
|
||||
try {
|
||||
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
|
||||
for (DataFileChunk chunk : this.chunks) {
|
||||
for (ArbitraryDataFileChunk chunk : this.chunks) {
|
||||
byte[] chunkHash = chunk.digest();
|
||||
if (chunkHash.length != 32) {
|
||||
LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
|
||||
@ -499,7 +499,7 @@ public class DataFile {
|
||||
public String printChunks() {
|
||||
String outputString = "";
|
||||
if (this.chunkCount() > 0) {
|
||||
for (DataFileChunk chunk : this.chunks) {
|
||||
for (ArbitraryDataFileChunk chunk : this.chunks) {
|
||||
if (outputString.length() > 0) {
|
||||
outputString = outputString.concat(",");
|
||||
}
|
||||
@ -10,24 +10,24 @@ import java.nio.file.Path;
|
||||
import java.nio.file.Paths;
|
||||
|
||||
|
||||
public class DataFileChunk extends DataFile {
|
||||
public class ArbitraryDataFileChunk extends ArbitraryDataFile {
|
||||
|
||||
private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class);
|
||||
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
|
||||
|
||||
public DataFileChunk(String hash58) {
|
||||
public ArbitraryDataFileChunk(String hash58) {
|
||||
super(hash58);
|
||||
}
|
||||
|
||||
public DataFileChunk(byte[] fileContent) {
|
||||
public ArbitraryDataFileChunk(byte[] fileContent) {
|
||||
super(fileContent);
|
||||
}
|
||||
|
||||
public static DataFileChunk fromHash58(String hash58) {
|
||||
return new DataFileChunk(hash58);
|
||||
public static ArbitraryDataFileChunk fromHash58(String hash58) {
|
||||
return new ArbitraryDataFileChunk(hash58);
|
||||
}
|
||||
|
||||
public static DataFileChunk fromHash(byte[] hash) {
|
||||
return DataFileChunk.fromHash58(Base58.encode(hash));
|
||||
public static ArbitraryDataFileChunk fromHash(byte[] hash) {
|
||||
return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash));
|
||||
}
|
||||
|
||||
@Override
|
||||
@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.*;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.repository.RepositoryManager;
|
||||
import org.qortal.storage.DataFile.*;
|
||||
import org.qortal.storage.ArbitraryDataFile.*;
|
||||
import org.qortal.transform.Transformer;
|
||||
import org.qortal.utils.Base58;
|
||||
import org.qortal.utils.FilesystemUtils;
|
||||
@ -165,9 +165,9 @@ public class ArbitraryDataReader {
|
||||
|
||||
private void fetchFromFileHash() {
|
||||
// Load data file directly from the hash
|
||||
DataFile dataFile = DataFile.fromHash58(resourceId);
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId);
|
||||
// Set filePath to the location of the DataFile
|
||||
this.filePath = Paths.get(dataFile.getFilePath());
|
||||
this.filePath = Paths.get(arbitraryDataFile.getFilePath());
|
||||
}
|
||||
|
||||
private void fetchFromName() throws IllegalStateException, IOException, DataException {
|
||||
@ -214,27 +214,27 @@ public class ArbitraryDataReader {
|
||||
}
|
||||
|
||||
// Load data file(s)
|
||||
DataFile dataFile = DataFile.fromHash(digest);
|
||||
if (!dataFile.exists()) {
|
||||
if (!dataFile.allChunksExist(chunkHashes)) {
|
||||
ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
|
||||
if (!arbitraryDataFile.exists()) {
|
||||
if (!arbitraryDataFile.allChunksExist(chunkHashes)) {
|
||||
// TODO: fetch them?
|
||||
throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile));
|
||||
throw new IllegalStateException(String.format("Missing chunks for file {}", arbitraryDataFile));
|
||||
}
|
||||
// We have all the chunks but not the complete file, so join them
|
||||
dataFile.addChunkHashes(chunkHashes);
|
||||
dataFile.join();
|
||||
arbitraryDataFile.addChunkHashes(chunkHashes);
|
||||
arbitraryDataFile.join();
|
||||
}
|
||||
|
||||
// If the complete file still doesn't exist then something went wrong
|
||||
if (!dataFile.exists()) {
|
||||
throw new IOException(String.format("File doesn't exist: %s", dataFile));
|
||||
if (!arbitraryDataFile.exists()) {
|
||||
throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile));
|
||||
}
|
||||
// Ensure the complete hash matches the joined chunks
|
||||
if (!Arrays.equals(dataFile.digest(), digest)) {
|
||||
if (!Arrays.equals(arbitraryDataFile.digest(), digest)) {
|
||||
throw new IllegalStateException("Unable to validate complete file hash");
|
||||
}
|
||||
// Set filePath to the location of the DataFile
|
||||
this.filePath = Paths.get(dataFile.getFilePath());
|
||||
this.filePath = Paths.get(arbitraryDataFile.getFilePath());
|
||||
}
|
||||
|
||||
private void decrypt() {
|
||||
|
||||
@ -6,7 +6,7 @@ import org.apache.logging.log4j.Logger;
|
||||
import org.qortal.data.transaction.ArbitraryTransactionData.*;
|
||||
import org.qortal.crypto.AES;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.storage.DataFile.*;
|
||||
import org.qortal.storage.ArbitraryDataFile.*;
|
||||
import org.qortal.utils.ZipUtils;
|
||||
|
||||
import javax.crypto.BadPaddingException;
|
||||
@ -33,7 +33,7 @@ public class ArbitraryDataWriter {
|
||||
private Compression compression;
|
||||
|
||||
private SecretKey aesKey;
|
||||
private DataFile dataFile;
|
||||
private ArbitraryDataFile arbitraryDataFile;
|
||||
|
||||
// Intermediate paths to cleanup
|
||||
private Path workingPath;
|
||||
@ -160,20 +160,20 @@ public class ArbitraryDataWriter {
|
||||
}
|
||||
|
||||
private void validate() throws IOException {
|
||||
if (this.dataFile == null) {
|
||||
if (this.arbitraryDataFile == null) {
|
||||
throw new IOException("No file available when validating");
|
||||
}
|
||||
this.dataFile.setSecret(this.aesKey.getEncoded());
|
||||
this.arbitraryDataFile.setSecret(this.aesKey.getEncoded());
|
||||
|
||||
// Validate the file
|
||||
ValidationResult validationResult = this.dataFile.isValid();
|
||||
ValidationResult validationResult = this.arbitraryDataFile.isValid();
|
||||
if (validationResult != ValidationResult.OK) {
|
||||
throw new IllegalStateException(String.format("File %s failed validation: %s", this.dataFile, validationResult));
|
||||
throw new IllegalStateException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult));
|
||||
}
|
||||
LOGGER.info("Whole file hash is valid: {}", this.dataFile.digest58());
|
||||
LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58());
|
||||
|
||||
// Validate each chunk
|
||||
for (DataFileChunk chunk : this.dataFile.getChunks()) {
|
||||
for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) {
|
||||
validationResult = chunk.isValid();
|
||||
if (validationResult != ValidationResult.OK) {
|
||||
throw new IllegalStateException(String.format("Chunk %s failed validation: %s", chunk, validationResult));
|
||||
@ -184,12 +184,12 @@ public class ArbitraryDataWriter {
|
||||
}
|
||||
|
||||
private void split() throws IOException {
|
||||
this.dataFile = DataFile.fromPath(this.filePath.toString());
|
||||
if (this.dataFile == null) {
|
||||
this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath.toString());
|
||||
if (this.arbitraryDataFile == null) {
|
||||
throw new IOException("No file available when trying to split");
|
||||
}
|
||||
|
||||
int chunkCount = this.dataFile.split(DataFile.CHUNK_SIZE);
|
||||
int chunkCount = this.arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
|
||||
if (chunkCount > 0) {
|
||||
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
|
||||
}
|
||||
@ -218,8 +218,8 @@ public class ArbitraryDataWriter {
|
||||
}
|
||||
|
||||
|
||||
public DataFile getDataFile() {
|
||||
return this.dataFile;
|
||||
public ArbitraryDataFile getArbitraryDataFile() {
|
||||
return this.arbitraryDataFile;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -13,8 +13,8 @@ import org.qortal.data.transaction.TransactionData;
|
||||
import org.qortal.payment.Payment;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.repository.Repository;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.DataFileChunk;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.storage.ArbitraryDataFileChunk;
|
||||
import org.qortal.transform.TransformationException;
|
||||
import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
|
||||
import org.qortal.transform.transaction.TransactionTransformer;
|
||||
@ -31,7 +31,7 @@ public class ArbitraryTransaction extends Transaction {
|
||||
public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes
|
||||
public static final int POW_MIN_DIFFICULTY = 12; // leading zero bits
|
||||
public static final int POW_MAX_DIFFICULTY = 19; // leading zero bits
|
||||
public static final long MAX_FILE_SIZE = DataFile.MAX_FILE_SIZE;
|
||||
public static final long MAX_FILE_SIZE = ArbitraryDataFile.MAX_FILE_SIZE;
|
||||
|
||||
// Constructors
|
||||
|
||||
@ -103,7 +103,7 @@ public class ArbitraryTransaction extends Transaction {
|
||||
}
|
||||
|
||||
// Check expected length of chunk hashes
|
||||
int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double)DataFileChunk.CHUNK_SIZE);
|
||||
int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double) ArbitraryDataFileChunk.CHUNK_SIZE);
|
||||
int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0;
|
||||
if (chunkHashes == null && expectedChunkHashesSize > 0) {
|
||||
return ValidationResult.INVALID_DATA_LENGTH;
|
||||
@ -121,7 +121,7 @@ public class ArbitraryTransaction extends Transaction {
|
||||
if (arbitraryTransactionData.getVersion() >= 5) {
|
||||
// Check reported length of the raw data
|
||||
// We should not download the raw data, so validation of that will be performed later
|
||||
if (arbitraryTransactionData.getSize() > DataFile.MAX_FILE_SIZE) {
|
||||
if (arbitraryTransactionData.getSize() > ArbitraryDataFile.MAX_FILE_SIZE) {
|
||||
return ValidationResult.INVALID_DATA_LENGTH;
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,7 @@ package org.qortal.test;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.qortal.repository.DataException;
|
||||
import org.qortal.storage.DataFile;
|
||||
import org.qortal.storage.ArbitraryDataFile;
|
||||
import org.qortal.test.common.Common;
|
||||
|
||||
import java.util.Random;
|
||||
@ -20,28 +20,28 @@ public class DataTests extends Common {
|
||||
@Test
|
||||
public void testSplitAndJoin() {
|
||||
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
|
||||
DataFile dataFile = new DataFile(dummyDataString.getBytes());
|
||||
assertTrue(dataFile.exists());
|
||||
assertEquals(62, dataFile.size());
|
||||
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58());
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes());
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(62, arbitraryDataFile.size());
|
||||
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
|
||||
|
||||
// Split into 7 chunks, each 10 bytes long
|
||||
dataFile.split(10);
|
||||
assertEquals(7, dataFile.chunkCount());
|
||||
arbitraryDataFile.split(10);
|
||||
assertEquals(7, arbitraryDataFile.chunkCount());
|
||||
|
||||
// Delete the original file
|
||||
dataFile.delete();
|
||||
assertFalse(dataFile.exists());
|
||||
assertEquals(0, dataFile.size());
|
||||
arbitraryDataFile.delete();
|
||||
assertFalse(arbitraryDataFile.exists());
|
||||
assertEquals(0, arbitraryDataFile.size());
|
||||
|
||||
// Now rebuild the original file from the chunks
|
||||
assertEquals(7, dataFile.chunkCount());
|
||||
dataFile.join();
|
||||
assertEquals(7, arbitraryDataFile.chunkCount());
|
||||
arbitraryDataFile.join();
|
||||
|
||||
// Validate that the original file is intact
|
||||
assertTrue(dataFile.exists());
|
||||
assertEquals(62, dataFile.size());
|
||||
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58());
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(62, arbitraryDataFile.size());
|
||||
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
|
||||
}
|
||||
|
||||
@Test
|
||||
@ -50,28 +50,28 @@ public class DataTests extends Common {
|
||||
byte[] randomData = new byte[fileSize];
|
||||
new Random().nextBytes(randomData); // No need for SecureRandom here
|
||||
|
||||
DataFile dataFile = new DataFile(randomData);
|
||||
assertTrue(dataFile.exists());
|
||||
assertEquals(fileSize, dataFile.size());
|
||||
String originalFileDigest = dataFile.digest58();
|
||||
ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData);
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(fileSize, arbitraryDataFile.size());
|
||||
String originalFileDigest = arbitraryDataFile.digest58();
|
||||
|
||||
// Split into chunks using 1MiB chunk size
|
||||
dataFile.split(1 * 1024 * 1024);
|
||||
assertEquals(6, dataFile.chunkCount());
|
||||
arbitraryDataFile.split(1 * 1024 * 1024);
|
||||
assertEquals(6, arbitraryDataFile.chunkCount());
|
||||
|
||||
// Delete the original file
|
||||
dataFile.delete();
|
||||
assertFalse(dataFile.exists());
|
||||
assertEquals(0, dataFile.size());
|
||||
arbitraryDataFile.delete();
|
||||
assertFalse(arbitraryDataFile.exists());
|
||||
assertEquals(0, arbitraryDataFile.size());
|
||||
|
||||
// Now rebuild the original file from the chunks
|
||||
assertEquals(6, dataFile.chunkCount());
|
||||
dataFile.join();
|
||||
assertEquals(6, arbitraryDataFile.chunkCount());
|
||||
arbitraryDataFile.join();
|
||||
|
||||
// Validate that the original file is intact
|
||||
assertTrue(dataFile.exists());
|
||||
assertEquals(fileSize, dataFile.size());
|
||||
assertEquals(originalFileDigest, dataFile.digest58());
|
||||
assertTrue(arbitraryDataFile.exists());
|
||||
assertEquals(fileSize, arbitraryDataFile.size());
|
||||
assertEquals(originalFileDigest, arbitraryDataFile.digest58());
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user