crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Renamed DataFile to ArbitraryDataFile, and DataFileChunk to ArbitraryDataFileChunk

Browse Source
This commit is contained in:
CalDescent 2021-08-14 13:25:45 +01:00
parent 16d93b1775
commit da6b341b63
12 changed files with 207 additions and 207 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
src/main/java/org/qortal/api/resource/ArbitraryResource.java
View File

@ -45,8 +45,8 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.storage.DataFileChunk; import org.qortal.storage.ArbitraryDataFileChunk;
import org.qortal.storage.ArbitraryDataWriter; import org.qortal.storage.ArbitraryDataWriter;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
@ -281,28 +281,28 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
DataFile dataFile = arbitraryDataWriter.getDataFile(); ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
if (dataFile == null) { if (arbitraryDataFile == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
DataFile.ValidationResult validationResult = dataFile.isValid(); ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid();
if (validationResult != DataFile.ValidationResult.OK) { if (validationResult != ArbitraryDataFile.ValidationResult.OK) {
LOGGER.error("Invalid file: {}", validationResult); LOGGER.error("Invalid file: {}", validationResult);
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
LOGGER.info("Whole file digest: {}", dataFile.digest58()); LOGGER.info("Whole file digest: {}", arbitraryDataFile.digest58());
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
if (chunkCount == 0) { if (chunkCount == 0) {
LOGGER.error("No chunks created"); LOGGER.error("No chunks created");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
String digest58 = dataFile.digest58(); String digest58 = arbitraryDataFile.digest58();
if (digest58 == null) { if (digest58 == null) {
LOGGER.error("Unable to calculate digest"); LOGGER.error("Unable to calculate digest");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
@ -313,12 +313,12 @@ public class ArbitraryResource {
final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP,
lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null);
final int size = (int)dataFile.size(); final int size = (int) arbitraryDataFile.size();
final int version = 5; final int version = 5;
final int nonce = 0; final int nonce = 0;
final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
final byte[] digest = dataFile.digest(); final byte[] digest = arbitraryDataFile.digest();
final byte[] chunkHashes = dataFile.chunkHashes(); final byte[] chunkHashes = arbitraryDataFile.chunkHashes();
final List<PaymentData> payments = new ArrayList<>(); final List<PaymentData> payments = new ArrayList<>();
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
@ -330,7 +330,7 @@ public class ArbitraryResource {
Transaction.ValidationResult result = transaction.isValidUnconfirmed(); Transaction.ValidationResult result = transaction.isValidUnconfirmed();
if (result != Transaction.ValidationResult.OK) { if (result != Transaction.ValidationResult.OK) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw TransactionsResource.createTransactionInvalidException(request, result); throw TransactionsResource.createTransactionInvalidException(request, result);
} }
@ -338,14 +338,14 @@ public class ArbitraryResource {
return Base58.encode(bytes); return Base58.encode(bytes);
} catch (DataException e) { } catch (DataException e) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
LOGGER.error("Repository issue when uploading data", e); LOGGER.error("Repository issue when uploading data", e);
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
} catch (TransformationException e) { } catch (TransformationException e) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
} catch (IllegalStateException e) { } catch (IllegalStateException e) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
LOGGER.error("Invalid upload data", e); LOGGER.error("Invalid upload data", e);
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA, e);
} }
@ -380,8 +380,8 @@ public class ArbitraryResource {
public String deleteFile(String hash58) { public String deleteFile(String hash58) {
Security.checkApiCallAllowed(request); Security.checkApiCallAllowed(request);
DataFile dataFile = DataFile.fromHash58(hash58); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58);
if (dataFile.delete()) { if (arbitraryDataFile.delete()) {
return "true"; return "true";
} }
return "false"; return "false";
@ -503,9 +503,9 @@ public class ArbitraryResource {
private boolean requestFile(String hash58, Peer targetPeer) { private boolean requestFile(String hash58, Peer targetPeer) {
try (final Repository repository = RepositoryManager.getRepository()) { try (final Repository repository = RepositoryManager.getRepository()) {
DataFile dataFile = DataFile.fromHash58(hash58); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(hash58);
if (dataFile.exists()) { if (arbitraryDataFile.exists()) {
LOGGER.info("Data file {} already exists but we'll request it anyway", dataFile); LOGGER.info("Data file {} already exists but we'll request it anyway", arbitraryDataFile);
} }
byte[] digest = null; byte[] digest = null;
@ -526,11 +526,11 @@ public class ArbitraryResource {
} }
DataFileMessage dataFileMessage = (DataFileMessage) message; DataFileMessage dataFileMessage = (DataFileMessage) message;
dataFile = dataFileMessage.getDataFile(); arbitraryDataFile = dataFileMessage.getArbitraryDataFile();
if (dataFile == null || !dataFile.exists()) { if (arbitraryDataFile == null || !arbitraryDataFile.exists()) {
return false; return false;
} }
LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getDataFile(), dataFileMessage.getDataFile().size())); LOGGER.info(String.format("Received file %s, size %d bytes", dataFileMessage.getArbitraryDataFile(), dataFileMessage.getArbitraryDataFile().size()));
return true; return true;
} catch (ApiException e) { } catch (ApiException e) {
throw e; throw e;
@ -571,22 +571,22 @@ public class ArbitraryResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_CRITERIA);
} }
DataFile dataFile = DataFile.fromHash58(combinedHash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(combinedHash);
if (dataFile.exists()) { if (arbitraryDataFile.exists()) {
LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash); LOGGER.info("We already have the combined file {}, but we'll join the chunks anyway.", combinedHash);
} }
String hash58List[] = files.split(","); String hash58List[] = files.split(",");
for (String hash58 : hash58List) { for (String hash58 : hash58List) {
if (hash58 != null) { if (hash58 != null) {
DataFileChunk chunk = DataFileChunk.fromHash58(hash58); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash58(hash58);
dataFile.addChunk(chunk); arbitraryDataFile.addChunk(chunk);
} }
} }
boolean success = dataFile.join(); boolean success = arbitraryDataFile.join();
if (success) { if (success) {
if (combinedHash.equals(dataFile.digest58())) { if (combinedHash.equals(arbitraryDataFile.digest58())) {
LOGGER.info("Valid hash {} after joining {} files", dataFile.digest58(), dataFile.chunkCount()); LOGGER.info("Valid hash {} after joining {} files", arbitraryDataFile.digest58(), arbitraryDataFile.chunkCount());
return Response.ok("true").build(); return Response.ok("true").build();
} }
} }

32
src/main/java/org/qortal/api/resource/WebsiteResource.java
View File

@ -38,8 +38,8 @@ import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.storage.DataFile.*; import org.qortal.storage.ArbitraryDataFile.*;
import org.qortal.storage.ArbitraryDataReader; import org.qortal.storage.ArbitraryDataReader;
import org.qortal.storage.ArbitraryDataWriter; import org.qortal.storage.ArbitraryDataWriter;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
@ -112,12 +112,12 @@ public class WebsiteResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
DataFile dataFile = arbitraryDataWriter.getDataFile(); ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
if (dataFile == null) { if (arbitraryDataFile == null) {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
String digest58 = dataFile.digest58(); String digest58 = arbitraryDataFile.digest58();
if (digest58 == null) { if (digest58 == null) {
LOGGER.error("Unable to calculate digest"); LOGGER.error("Unable to calculate digest");
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
@ -130,13 +130,13 @@ public class WebsiteResource {
final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP, final BaseTransactionData baseTransactionData = new BaseTransactionData(NTP.getTime(), Group.NO_GROUP,
lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null); lastReference, creatorPublicKey, BlockChain.getInstance().getUnitFee(), null);
final int size = (int)dataFile.size(); final int size = (int) arbitraryDataFile.size();
final int version = 5; final int version = 5;
final int nonce = 0; final int nonce = 0;
byte[] secret = dataFile.getSecret(); byte[] secret = arbitraryDataFile.getSecret();
final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH; final ArbitraryTransactionData.DataType dataType = ArbitraryTransactionData.DataType.DATA_HASH;
final byte[] digest = dataFile.digest(); final byte[] digest = arbitraryDataFile.digest();
final byte[] chunkHashes = dataFile.chunkHashes(); final byte[] chunkHashes = arbitraryDataFile.chunkHashes();
final List<PaymentData> payments = new ArrayList<>(); final List<PaymentData> payments = new ArrayList<>();
ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData, ArbitraryTransactionData transactionData = new ArbitraryTransactionData(baseTransactionData,
@ -149,7 +149,7 @@ public class WebsiteResource {
Transaction.ValidationResult result = transaction.isValidUnconfirmed(); Transaction.ValidationResult result = transaction.isValidUnconfirmed();
if (result != Transaction.ValidationResult.OK) { if (result != Transaction.ValidationResult.OK) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw TransactionsResource.createTransactionInvalidException(request, result); throw TransactionsResource.createTransactionInvalidException(request, result);
} }
@ -157,10 +157,10 @@ public class WebsiteResource {
return Base58.encode(bytes); return Base58.encode(bytes);
} catch (TransformationException e) { } catch (TransformationException e) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.TRANSFORMATION_ERROR, e);
} catch (DataException e) { } catch (DataException e) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.REPOSITORY_ISSUE, e);
} }
} }
@ -212,11 +212,11 @@ public class WebsiteResource {
throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA); throw ApiExceptionFactory.INSTANCE.createException(request, ApiError.INVALID_DATA);
} }
DataFile dataFile = arbitraryDataWriter.getDataFile(); ArbitraryDataFile arbitraryDataFile = arbitraryDataWriter.getArbitraryDataFile();
if (dataFile != null) { if (arbitraryDataFile != null) {
String digest58 = dataFile.digest58(); String digest58 = arbitraryDataFile.digest58();
if (digest58 != null) { if (digest58 != null) {
return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(dataFile.getSecret()); return "http://localhost:12393/site/hash/" + digest58 + "?secret=" + Base58.encode(arbitraryDataFile.getSecret());
} }
} }
return "Unable to generate preview URL"; return "Unable to generate preview URL";

40
src/main/java/org/qortal/controller/ArbitraryDataManager.java
View File

@ -13,8 +13,8 @@ import org.qortal.network.message.*;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.storage.DataFileChunk; import org.qortal.storage.ArbitraryDataFileChunk;
import org.qortal.transaction.ArbitraryTransaction; import org.qortal.transaction.ArbitraryTransaction;
import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
@ -166,7 +166,7 @@ public class ArbitraryDataManager extends Thread {
return true; return true;
} }
private DataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException { private ArbitraryDataFile fetchArbitraryDataFile(Peer peer, byte[] hash) throws InterruptedException {
String hash58 = Base58.encode(hash); String hash58 = Base58.encode(hash);
LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer)); LOGGER.info(String.format("Fetching data file %.8s from peer %s", hash58, peer));
arbitraryDataFileRequests.put(hash58, NTP.getTime()); arbitraryDataFileRequests.put(hash58, NTP.getTime());
@ -181,7 +181,7 @@ public class ArbitraryDataManager extends Thread {
} }
DataFileMessage dataFileMessage = (DataFileMessage) message; DataFileMessage dataFileMessage = (DataFileMessage) message;
return dataFileMessage.getDataFile(); return dataFileMessage.getArbitraryDataFile();
} }
public void cleanupRequestCache(long now) { public void cleanupRequestCache(long now) {
@ -269,13 +269,13 @@ public class ArbitraryDataManager extends Thread {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Load data file(s) // Load data file(s)
DataFile dataFile = DataFile.fromHash(arbitraryTransactionData.getData()); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(arbitraryTransactionData.getData());
dataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes()); arbitraryDataFile.addChunkHashes(arbitraryTransactionData.getChunkHashes());
// Check all hashes exist // Check all hashes exist
for (byte[] hash : hashes) { for (byte[] hash : hashes) {
//LOGGER.info("Received hash {}", Base58.encode(hash)); //LOGGER.info("Received hash {}", Base58.encode(hash));
if (!dataFile.containsChunk(hash)) { if (!arbitraryDataFile.containsChunk(hash)) {
LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58); LOGGER.info("Received non-matching chunk hash {} for signature {}", Base58.encode(hash), signature58);
return; return;
} }
@ -287,14 +287,14 @@ public class ArbitraryDataManager extends Thread {
// Now fetch actual data from this peer // Now fetch actual data from this peer
for (byte[] hash : hashes) { for (byte[] hash : hashes) {
if (!dataFile.chunkExists(hash)) { if (!arbitraryDataFile.chunkExists(hash)) {
// Only request the file if we aren't already requesting it from someone else // Only request the file if we aren't already requesting it from someone else
if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) { if (!arbitraryDataFileRequests.containsKey(Base58.encode(hash))) {
DataFile receivedDataFile = fetchArbitraryDataFile(peer, hash); ArbitraryDataFile receivedArbitraryDataFile = fetchArbitraryDataFile(peer, hash);
LOGGER.info("Received data file {} from peer {}", receivedDataFile, peer); LOGGER.info("Received data file {} from peer {}", receivedArbitraryDataFile, peer);
} }
else { else {
LOGGER.info("Already requesting data file {}", dataFile); LOGGER.info("Already requesting data file {}", arbitraryDataFile);
} }
} }
} }
@ -318,15 +318,15 @@ public class ArbitraryDataManager extends Thread {
byte[] hash = getDataFileMessage.getHash(); byte[] hash = getDataFileMessage.getHash();
Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet(); Controller.getInstance().stats.getDataFileMessageStats.requests.incrementAndGet();
DataFile dataFile = DataFile.fromHash(hash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
if (dataFile.exists()) { if (arbitraryDataFile.exists()) {
DataFileMessage dataFileMessage = new DataFileMessage(dataFile); DataFileMessage dataFileMessage = new DataFileMessage(arbitraryDataFile);
dataFileMessage.setId(message.getId()); dataFileMessage.setId(message.getId());
if (!peer.sendMessage(dataFileMessage)) { if (!peer.sendMessage(dataFileMessage)) {
LOGGER.info("Couldn't sent file"); LOGGER.info("Couldn't sent file");
peer.disconnect("failed to send file"); peer.disconnect("failed to send file");
} }
LOGGER.info("Sent file {}", dataFile); LOGGER.info("Sent file {}", arbitraryDataFile);
} }
else { else {
@ -334,7 +334,7 @@ public class ArbitraryDataManager extends Thread {
Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement(); Controller.getInstance().stats.getDataFileMessageStats.unknownFiles.getAndIncrement();
// Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout // Send valid, yet unexpected message type in response, so peer's synchronizer doesn't have to wait for timeout
LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, dataFile)); LOGGER.debug(() -> String.format("Sending 'file unknown' response to peer %s for GET_FILE request for unknown file %s", peer, arbitraryDataFile));
// We'll send empty block summaries message as it's very short // We'll send empty block summaries message as it's very short
// TODO: use a different message type here // TODO: use a different message type here
@ -344,7 +344,7 @@ public class ArbitraryDataManager extends Thread {
LOGGER.info("Couldn't sent file-unknown response"); LOGGER.info("Couldn't sent file-unknown response");
peer.disconnect("failed to send file-unknown response"); peer.disconnect("failed to send file-unknown response");
} }
LOGGER.info("Sent file-unknown response for file {}", dataFile); LOGGER.info("Sent file-unknown response for file {}", arbitraryDataFile);
} }
} }
@ -367,10 +367,10 @@ public class ArbitraryDataManager extends Thread {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load file(s) and add any that exist to the list of hashes // Load file(s) and add any that exist to the list of hashes
DataFile dataFile = DataFile.fromHash(hash); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(hash);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
for (DataFileChunk dataFileChunk : dataFile.getChunks()) { for (ArbitraryDataFileChunk dataFileChunk : arbitraryDataFile.getChunks()) {
if (dataFileChunk.exists()) { if (dataFileChunk.exists()) {
hashes.add(dataFileChunk.getHash()); hashes.add(dataFileChunk.getHash());
//LOGGER.info("Added hash {}", dataFileChunk.getHash58()); //LOGGER.info("Added hash {}", dataFileChunk.getHash58());

26
src/main/java/org/qortal/network/message/DataFileMessage.java
View File

@ -1,7 +1,7 @@
package org.qortal.network.message; package org.qortal.network.message;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
@ -10,22 +10,22 @@ import java.nio.ByteBuffer;
public class DataFileMessage extends Message { public class DataFileMessage extends Message {
private final DataFile dataFile; private final ArbitraryDataFile arbitraryDataFile;
public DataFileMessage(DataFile dataFile) { public DataFileMessage(ArbitraryDataFile arbitraryDataFile) {
super(MessageType.DATA_FILE); super(MessageType.DATA_FILE);
this.dataFile = dataFile; this.arbitraryDataFile = arbitraryDataFile;
} }
public DataFileMessage(int id, DataFile dataFile) { public DataFileMessage(int id, ArbitraryDataFile arbitraryDataFile) {
super(id, MessageType.DATA_FILE); super(id, MessageType.DATA_FILE);
this.dataFile = dataFile; this.arbitraryDataFile = arbitraryDataFile;
} }
public DataFile getDataFile() { public ArbitraryDataFile getArbitraryDataFile() {
return this.dataFile; return this.arbitraryDataFile;
} }
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
@ -36,18 +36,18 @@ public class DataFileMessage extends Message {
byte[] data = new byte[dataLength]; byte[] data = new byte[dataLength];
byteBuffer.get(data); byteBuffer.get(data);
DataFile dataFile = new DataFile(data); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(data);
return new DataFileMessage(id, dataFile); return new DataFileMessage(id, arbitraryDataFile);
} }
@Override @Override
protected byte[] toData() { protected byte[] toData() {
if (this.dataFile == null) { if (this.arbitraryDataFile == null) {
return null; return null;
} }
byte[] data = this.dataFile.getBytes(); byte[] data = this.arbitraryDataFile.getBytes();
if (data == null) { if (data == null) {
return null; return null;
} }
@ -66,7 +66,7 @@ public class DataFileMessage extends Message {
} }
public DataFileMessage cloneWithNewId(int newId) { public DataFileMessage cloneWithNewId(int newId) {
DataFileMessage clone = new DataFileMessage(this.dataFile); DataFileMessage clone = new DataFileMessage(this.arbitraryDataFile);
clone.setId(newId); clone.setId(newId);
return clone; return clone;
} }

52
src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -10,7 +10,7 @@ import org.qortal.data.transaction.BaseTransactionData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.ArbitraryRepository; import org.qortal.repository.ArbitraryRepository;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.transaction.Transaction.ApprovalStatus; import org.qortal.transaction.Transaction.ApprovalStatus;
import java.sql.ResultSet; import java.sql.ResultSet;
@ -55,18 +55,18 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
DataFile dataFile = DataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
// Check if we already have the complete data file // Check if we already have the complete data file
if (dataFile.exists()) { if (arbitraryDataFile.exists()) {
return true; return true;
} }
// Alternatively, if we have all the chunks, then it's safe to assume the data is local // Alternatively, if we have all the chunks, then it's safe to assume the data is local
if (dataFile.allChunksExist(chunkHashes)) { if (arbitraryDataFile.allChunksExist(chunkHashes)) {
return true; return true;
} }
@ -90,23 +90,23 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = transactionData.getChunkHashes(); byte[] chunkHashes = transactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
DataFile dataFile = DataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
// If we have the complete data file, return it // If we have the complete data file, return it
if (dataFile.exists()) { if (arbitraryDataFile.exists()) {
return dataFile.getBytes(); return arbitraryDataFile.getBytes();
} }
// Alternatively, if we have all the chunks, combine them into a single file // Alternatively, if we have all the chunks, combine them into a single file
if (dataFile.allChunksExist(chunkHashes)) { if (arbitraryDataFile.allChunksExist(chunkHashes)) {
dataFile.join(); arbitraryDataFile.join();
// Verify that the combined hash matches the expected hash // Verify that the combined hash matches the expected hash
if (digest.equals(dataFile.digest())) { if (digest.equals(arbitraryDataFile.digest())) {
return dataFile.getBytes(); return arbitraryDataFile.getBytes();
} }
} }
@ -134,29 +134,29 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
arbitraryTransactionData.setDataType(DataType.DATA_HASH); arbitraryTransactionData.setDataType(DataType.DATA_HASH);
// Create DataFile // Create DataFile
DataFile dataFile = new DataFile(rawData); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(rawData);
// Verify that the data file is valid, and that it matches the expected hash // Verify that the data file is valid, and that it matches the expected hash
DataFile.ValidationResult validationResult = dataFile.isValid(); ArbitraryDataFile.ValidationResult validationResult = arbitraryDataFile.isValid();
if (validationResult != DataFile.ValidationResult.OK) { if (validationResult != ArbitraryDataFile.ValidationResult.OK) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw new DataException("Invalid data file when attempting to store arbitrary transaction data"); throw new DataException("Invalid data file when attempting to store arbitrary transaction data");
} }
if (!dataHash.equals(dataFile.digest())) { if (!dataHash.equals(arbitraryDataFile.digest())) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw new DataException("Could not verify hash when attempting to store arbitrary transaction data"); throw new DataException("Could not verify hash when attempting to store arbitrary transaction data");
} }
// Now create chunks if needed // Now create chunks if needed
int chunkCount = dataFile.split(DataFile.CHUNK_SIZE); int chunkCount = arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
if (chunkCount > 0) { if (chunkCount > 0) {
LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s")));
LOGGER.info("{}", dataFile.printChunks()); LOGGER.info("{}", arbitraryDataFile.printChunks());
// Verify that the chunk hashes match those in the transaction // Verify that the chunk hashes match those in the transaction
byte[] chunkHashes = dataFile.chunkHashes(); byte[] chunkHashes = arbitraryDataFile.chunkHashes();
if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) { if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) {
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data"); throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data");
} }
@ -175,13 +175,13 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
byte[] chunkHashes = arbitraryTransactionData.getChunkHashes(); byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
// Load data file(s) // Load data file(s)
DataFile dataFile = DataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
if (chunkHashes != null && chunkHashes.length > 0) { if (chunkHashes != null && chunkHashes.length > 0) {
dataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
} }
// Delete file and chunks // Delete file and chunks
dataFile.deleteAll(); arbitraryDataFile.deleteAll();
} }
@Override @Override

2
src/main/java/org/qortal/storage/ArbitraryDataBuilder.java
View File

@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.Service;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.storage.DataFile.ResourceIdType; import org.qortal.storage.ArbitraryDataFile.ResourceIdType;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import java.io.IOException; import java.io.IOException;

64
src/main/java/org/qortal/storage/DataFile.java → src/main/java/org/qortal/storage/ArbitraryDataFile.java
View File

@ -20,7 +20,7 @@ import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toMap;
public class DataFile { public class ArbitraryDataFile {
// Validation results // Validation results
public enum ValidationResult { public enum ValidationResult {
@ -30,13 +30,13 @@ public class DataFile {
public final int value; public final int value;
private static final Map<Integer, DataFile.ValidationResult> map = stream(DataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result)); private static final Map<Integer, ArbitraryDataFile.ValidationResult> map = stream(ArbitraryDataFile.ValidationResult.values()).collect(toMap(result -> result.value, result -> result));
ValidationResult(int value) { ValidationResult(int value) {
this.value = value; this.value = value;
} }
public static DataFile.ValidationResult valueOf(int value) { public static ArbitraryDataFile.ValidationResult valueOf(int value) {
return map.get(value); return map.get(value);
} }
} }
@ -49,7 +49,7 @@ public class DataFile {
NAME NAME
}; };
private static final Logger LOGGER = LogManager.getLogger(DataFile.class); private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFile.class);
public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB public static final long MAX_FILE_SIZE = 500 * 1024 * 1024; // 500MiB
public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB public static final int CHUNK_SIZE = 1 * 1024 * 1024; // 1MiB
@ -57,20 +57,20 @@ public class DataFile {
protected String filePath; protected String filePath;
protected String hash58; protected String hash58;
private ArrayList<DataFileChunk> chunks; private ArrayList<ArbitraryDataFileChunk> chunks;
private byte[] secret; private byte[] secret;
public DataFile() { public ArbitraryDataFile() {
} }
public DataFile(String hash58) { public ArbitraryDataFile(String hash58) {
this.createDataDirectory(); this.createDataDirectory();
this.filePath = DataFile.getOutputFilePath(hash58, false); this.filePath = ArbitraryDataFile.getOutputFilePath(hash58, false);
this.chunks = new ArrayList<>(); this.chunks = new ArrayList<>();
this.hash58 = hash58; this.hash58 = hash58;
} }
public DataFile(byte[] fileContent) { public ArbitraryDataFile(byte[] fileContent) {
if (fileContent == null) { if (fileContent == null) {
LOGGER.error("fileContent is null"); LOGGER.error("fileContent is null");
return; return;
@ -95,28 +95,28 @@ public class DataFile {
} }
} }
public static DataFile fromHash58(String hash58) { public static ArbitraryDataFile fromHash58(String hash58) {
return new DataFile(hash58); return new ArbitraryDataFile(hash58);
} }
public static DataFile fromHash(byte[] hash) { public static ArbitraryDataFile fromHash(byte[] hash) {
return DataFile.fromHash58(Base58.encode(hash)); return ArbitraryDataFile.fromHash58(Base58.encode(hash));
} }
public static DataFile fromPath(String path) { public static ArbitraryDataFile fromPath(String path) {
File file = new File(path); File file = new File(path);
if (file.exists()) { if (file.exists()) {
try { try {
byte[] fileContent = Files.readAllBytes(file.toPath()); byte[] fileContent = Files.readAllBytes(file.toPath());
byte[] digest = Crypto.digest(fileContent); byte[] digest = Crypto.digest(fileContent);
DataFile dataFile = DataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
// Copy file to base directory if needed // Copy file to base directory if needed
Path filePath = Paths.get(path); Path filePath = Paths.get(path);
if (Files.exists(filePath) && !dataFile.isInBaseDirectory(path)) { if (Files.exists(filePath) && !arbitraryDataFile.isInBaseDirectory(path)) {
dataFile.copyToDataDirectory(filePath); arbitraryDataFile.copyToDataDirectory(filePath);
} }
return dataFile; return arbitraryDataFile;
} catch (IOException e) { } catch (IOException e) {
LOGGER.error("Couldn't compute digest for DataFile"); LOGGER.error("Couldn't compute digest for DataFile");
@ -125,8 +125,8 @@ public class DataFile {
return null; return null;
} }
public static DataFile fromFile(File file) { public static ArbitraryDataFile fromFile(File file) {
return DataFile.fromPath(file.getPath()); return ArbitraryDataFile.fromPath(file.getPath());
} }
private boolean createDataDirectory() { private boolean createDataDirectory() {
@ -188,7 +188,7 @@ public class DataFile {
long fileSize = Files.size(path); long fileSize = Files.size(path);
if (fileSize > MAX_FILE_SIZE) { if (fileSize > MAX_FILE_SIZE) {
LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE)); LOGGER.error(String.format("DataFile is too large: %d bytes (max size: %d bytes)", fileSize, MAX_FILE_SIZE));
return DataFile.ValidationResult.FILE_TOO_LARGE; return ArbitraryDataFile.ValidationResult.FILE_TOO_LARGE;
} }
} catch (IOException e) { } catch (IOException e) {
@ -198,7 +198,7 @@ public class DataFile {
return ValidationResult.OK; return ValidationResult.OK;
} }
public void addChunk(DataFileChunk chunk) { public void addChunk(ArbitraryDataFileChunk chunk) {
this.chunks.add(chunk); this.chunks.add(chunk);
} }
@ -210,7 +210,7 @@ public class DataFile {
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkDigest = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkDigest); byteBuffer.get(chunkDigest);
DataFileChunk chunk = DataFileChunk.fromHash(chunkDigest); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkDigest);
this.addChunk(chunk); this.addChunk(chunk);
} }
} }
@ -232,7 +232,7 @@ public class DataFile {
out.write(buffer, 0, numberOfBytes); out.write(buffer, 0, numberOfBytes);
out.flush(); out.flush();
DataFileChunk chunk = new DataFileChunk(out.toByteArray()); ArbitraryDataFileChunk chunk = new ArbitraryDataFileChunk(out.toByteArray());
ValidationResult validationResult = chunk.isValid(); ValidationResult validationResult = chunk.isValid();
if (validationResult == ValidationResult.OK) { if (validationResult == ValidationResult.OK) {
this.chunks.add(chunk); this.chunks.add(chunk);
@ -266,7 +266,7 @@ public class DataFile {
// Join the chunks // Join the chunks
File outputFile = new File(this.filePath); File outputFile = new File(this.filePath);
try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) { try (BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(outputFile))) {
for (DataFileChunk chunk : this.chunks) { for (ArbitraryDataFileChunk chunk : this.chunks) {
File sourceFile = new File(chunk.filePath); File sourceFile = new File(chunk.filePath);
BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); BufferedInputStream in = new BufferedInputStream(new FileInputStream(sourceFile));
byte[] buffer = new byte[2048]; byte[] buffer = new byte[2048];
@ -323,7 +323,7 @@ public class DataFile {
if (this.chunks != null && this.chunks.size() > 0) { if (this.chunks != null && this.chunks.size() > 0) {
Iterator iterator = this.chunks.iterator(); Iterator iterator = this.chunks.iterator();
while (iterator.hasNext()) { while (iterator.hasNext()) {
DataFileChunk chunk = (DataFileChunk) iterator.next(); ArbitraryDataFileChunk chunk = (ArbitraryDataFileChunk) iterator.next();
chunk.delete(); chunk.delete();
iterator.remove(); iterator.remove();
success = true; success = true;
@ -376,7 +376,7 @@ public class DataFile {
} }
public boolean chunkExists(byte[] hash) { public boolean chunkExists(byte[] hash) {
for (DataFileChunk chunk : this.chunks) { for (ArbitraryDataFileChunk chunk : this.chunks) {
if (Arrays.equals(hash, chunk.getHash())) { if (Arrays.equals(hash, chunk.getHash())) {
return chunk.exists(); return chunk.exists();
} }
@ -389,7 +389,7 @@ public class DataFile {
while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) { while (byteBuffer.remaining() >= TransactionTransformer.SHA256_LENGTH) {
byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH]; byte[] chunkHash = new byte[TransactionTransformer.SHA256_LENGTH];
byteBuffer.get(chunkHash); byteBuffer.get(chunkHash);
DataFileChunk chunk = DataFileChunk.fromHash(chunkHash); ArbitraryDataFileChunk chunk = ArbitraryDataFileChunk.fromHash(chunkHash);
if (!chunk.exists()) { if (!chunk.exists()) {
return false; return false;
} }
@ -398,7 +398,7 @@ public class DataFile {
} }
public boolean containsChunk(byte[] hash) { public boolean containsChunk(byte[] hash) {
for (DataFileChunk chunk : this.chunks) { for (ArbitraryDataFileChunk chunk : this.chunks) {
if (Arrays.equals(hash, chunk.getHash())) { if (Arrays.equals(hash, chunk.getHash())) {
return true; return true;
} }
@ -419,7 +419,7 @@ public class DataFile {
return this.chunks.size(); return this.chunks.size();
} }
public List<DataFileChunk> getChunks() { public List<ArbitraryDataFileChunk> getChunks() {
return this.chunks; return this.chunks;
} }
@ -432,7 +432,7 @@ public class DataFile {
try { try {
ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
for (DataFileChunk chunk : this.chunks) { for (ArbitraryDataFileChunk chunk : this.chunks) {
byte[] chunkHash = chunk.digest(); byte[] chunkHash = chunk.digest();
if (chunkHash.length != 32) { if (chunkHash.length != 32) {
LOGGER.info("Invalid chunk hash length: {}", chunkHash.length); LOGGER.info("Invalid chunk hash length: {}", chunkHash.length);
@ -499,7 +499,7 @@ public class DataFile {
public String printChunks() { public String printChunks() {
String outputString = ""; String outputString = "";
if (this.chunkCount() > 0) { if (this.chunkCount() > 0) {
for (DataFileChunk chunk : this.chunks) { for (ArbitraryDataFileChunk chunk : this.chunks) {
if (outputString.length() > 0) { if (outputString.length() > 0) {
outputString = outputString.concat(","); outputString = outputString.concat(",");
} }

16
src/main/java/org/qortal/storage/DataFileChunk.java → src/main/java/org/qortal/storage/ArbitraryDataFileChunk.java
View File

@ -10,24 +10,24 @@ import java.nio.file.Path;
import java.nio.file.Paths; import java.nio.file.Paths;
public class DataFileChunk extends DataFile { public class ArbitraryDataFileChunk extends ArbitraryDataFile {
private static final Logger LOGGER = LogManager.getLogger(DataFileChunk.class); private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataFileChunk.class);
public DataFileChunk(String hash58) { public ArbitraryDataFileChunk(String hash58) {
super(hash58); super(hash58);
} }
public DataFileChunk(byte[] fileContent) { public ArbitraryDataFileChunk(byte[] fileContent) {
super(fileContent); super(fileContent);
} }
public static DataFileChunk fromHash58(String hash58) { public static ArbitraryDataFileChunk fromHash58(String hash58) {
return new DataFileChunk(hash58); return new ArbitraryDataFileChunk(hash58);
} }
public static DataFileChunk fromHash(byte[] hash) { public static ArbitraryDataFileChunk fromHash(byte[] hash) {
return DataFileChunk.fromHash58(Base58.encode(hash)); return ArbitraryDataFileChunk.fromHash58(Base58.encode(hash));
} }
@Override @Override

26
src/main/java/org/qortal/storage/ArbitraryDataReader.java
View File

@ -8,7 +8,7 @@ import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
import org.qortal.storage.DataFile.*; import org.qortal.storage.ArbitraryDataFile.*;
import org.qortal.transform.Transformer; import org.qortal.transform.Transformer;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import org.qortal.utils.FilesystemUtils; import org.qortal.utils.FilesystemUtils;
@ -165,9 +165,9 @@ public class ArbitraryDataReader {
private void fetchFromFileHash() { private void fetchFromFileHash() {
// Load data file directly from the hash // Load data file directly from the hash
DataFile dataFile = DataFile.fromHash58(resourceId); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash58(resourceId);
// Set filePath to the location of the DataFile // Set filePath to the location of the DataFile
this.filePath = Paths.get(dataFile.getFilePath()); this.filePath = Paths.get(arbitraryDataFile.getFilePath());
} }
private void fetchFromName() throws IllegalStateException, IOException, DataException { private void fetchFromName() throws IllegalStateException, IOException, DataException {
@ -214,27 +214,27 @@ public class ArbitraryDataReader {
} }
// Load data file(s) // Load data file(s)
DataFile dataFile = DataFile.fromHash(digest); ArbitraryDataFile arbitraryDataFile = ArbitraryDataFile.fromHash(digest);
if (!dataFile.exists()) { if (!arbitraryDataFile.exists()) {
if (!dataFile.allChunksExist(chunkHashes)) { if (!arbitraryDataFile.allChunksExist(chunkHashes)) {
// TODO: fetch them? // TODO: fetch them?
throw new IllegalStateException(String.format("Missing chunks for file {}", dataFile)); throw new IllegalStateException(String.format("Missing chunks for file {}", arbitraryDataFile));
} }
// We have all the chunks but not the complete file, so join them // We have all the chunks but not the complete file, so join them
dataFile.addChunkHashes(chunkHashes); arbitraryDataFile.addChunkHashes(chunkHashes);
dataFile.join(); arbitraryDataFile.join();
} }
// If the complete file still doesn't exist then something went wrong // If the complete file still doesn't exist then something went wrong
if (!dataFile.exists()) { if (!arbitraryDataFile.exists()) {
throw new IOException(String.format("File doesn't exist: %s", dataFile)); throw new IOException(String.format("File doesn't exist: %s", arbitraryDataFile));
} }
// Ensure the complete hash matches the joined chunks // Ensure the complete hash matches the joined chunks
if (!Arrays.equals(dataFile.digest(), digest)) { if (!Arrays.equals(arbitraryDataFile.digest(), digest)) {
throw new IllegalStateException("Unable to validate complete file hash"); throw new IllegalStateException("Unable to validate complete file hash");
} }
// Set filePath to the location of the DataFile // Set filePath to the location of the DataFile
this.filePath = Paths.get(dataFile.getFilePath()); this.filePath = Paths.get(arbitraryDataFile.getFilePath());
} }
private void decrypt() { private void decrypt() {

26
src/main/java/org/qortal/storage/ArbitraryDataWriter.java
View File

@ -6,7 +6,7 @@ import org.apache.logging.log4j.Logger;
import org.qortal.data.transaction.ArbitraryTransactionData.*; import org.qortal.data.transaction.ArbitraryTransactionData.*;
import org.qortal.crypto.AES; import org.qortal.crypto.AES;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.storage.DataFile.*; import org.qortal.storage.ArbitraryDataFile.*;
import org.qortal.utils.ZipUtils; import org.qortal.utils.ZipUtils;
import javax.crypto.BadPaddingException; import javax.crypto.BadPaddingException;
@ -33,7 +33,7 @@ public class ArbitraryDataWriter {
private Compression compression; private Compression compression;
private SecretKey aesKey; private SecretKey aesKey;
private DataFile dataFile; private ArbitraryDataFile arbitraryDataFile;
// Intermediate paths to cleanup // Intermediate paths to cleanup
private Path workingPath; private Path workingPath;
@ -160,20 +160,20 @@ public class ArbitraryDataWriter {
} }
private void validate() throws IOException { private void validate() throws IOException {
if (this.dataFile == null) { if (this.arbitraryDataFile == null) {
throw new IOException("No file available when validating"); throw new IOException("No file available when validating");
} }
this.dataFile.setSecret(this.aesKey.getEncoded()); this.arbitraryDataFile.setSecret(this.aesKey.getEncoded());
// Validate the file // Validate the file
ValidationResult validationResult = this.dataFile.isValid(); ValidationResult validationResult = this.arbitraryDataFile.isValid();
if (validationResult != ValidationResult.OK) { if (validationResult != ValidationResult.OK) {
throw new IllegalStateException(String.format("File %s failed validation: %s", this.dataFile, validationResult)); throw new IllegalStateException(String.format("File %s failed validation: %s", this.arbitraryDataFile, validationResult));
} }
LOGGER.info("Whole file hash is valid: {}", this.dataFile.digest58()); LOGGER.info("Whole file hash is valid: {}", this.arbitraryDataFile.digest58());
// Validate each chunk // Validate each chunk
for (DataFileChunk chunk : this.dataFile.getChunks()) { for (ArbitraryDataFileChunk chunk : this.arbitraryDataFile.getChunks()) {
validationResult = chunk.isValid(); validationResult = chunk.isValid();
if (validationResult != ValidationResult.OK) { if (validationResult != ValidationResult.OK) {
throw new IllegalStateException(String.format("Chunk %s failed validation: %s", chunk, validationResult)); throw new IllegalStateException(String.format("Chunk %s failed validation: %s", chunk, validationResult));
@ -184,12 +184,12 @@ public class ArbitraryDataWriter {
} }
private void split() throws IOException { private void split() throws IOException {
this.dataFile = DataFile.fromPath(this.filePath.toString()); this.arbitraryDataFile = ArbitraryDataFile.fromPath(this.filePath.toString());
if (this.dataFile == null) { if (this.arbitraryDataFile == null) {
throw new IOException("No file available when trying to split"); throw new IOException("No file available when trying to split");
} }
int chunkCount = this.dataFile.split(DataFile.CHUNK_SIZE); int chunkCount = this.arbitraryDataFile.split(ArbitraryDataFile.CHUNK_SIZE);
if (chunkCount > 0) { if (chunkCount > 0) {
LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s"))); LOGGER.info(String.format("Successfully split into %d chunk%s", chunkCount, (chunkCount == 1 ? "" : "s")));
} }
@ -218,8 +218,8 @@ public class ArbitraryDataWriter {
} }
public DataFile getDataFile() { public ArbitraryDataFile getArbitraryDataFile() {
return this.dataFile; return this.arbitraryDataFile;
} }
} }

10
src/main/java/org/qortal/transaction/ArbitraryTransaction.java
View File

@ -13,8 +13,8 @@ import org.qortal.data.transaction.TransactionData;
import org.qortal.payment.Payment; import org.qortal.payment.Payment;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.storage.DataFileChunk; import org.qortal.storage.ArbitraryDataFileChunk;
import org.qortal.transform.TransformationException; import org.qortal.transform.TransformationException;
import org.qortal.transform.transaction.ArbitraryTransactionTransformer; import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer;
@ -31,7 +31,7 @@ public class ArbitraryTransaction extends Transaction {
public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes
public static final int POW_MIN_DIFFICULTY = 12; // leading zero bits public static final int POW_MIN_DIFFICULTY = 12; // leading zero bits
public static final int POW_MAX_DIFFICULTY = 19; // leading zero bits public static final int POW_MAX_DIFFICULTY = 19; // leading zero bits
public static final long MAX_FILE_SIZE = DataFile.MAX_FILE_SIZE; public static final long MAX_FILE_SIZE = ArbitraryDataFile.MAX_FILE_SIZE;
// Constructors // Constructors
@ -103,7 +103,7 @@ public class ArbitraryTransaction extends Transaction {
} }
// Check expected length of chunk hashes // Check expected length of chunk hashes
int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double)DataFileChunk.CHUNK_SIZE); int chunkCount = (int)Math.ceil((double)arbitraryTransactionData.getSize() / (double) ArbitraryDataFileChunk.CHUNK_SIZE);
int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0; int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0;
if (chunkHashes == null && expectedChunkHashesSize > 0) { if (chunkHashes == null && expectedChunkHashesSize > 0) {
return ValidationResult.INVALID_DATA_LENGTH; return ValidationResult.INVALID_DATA_LENGTH;
@ -121,7 +121,7 @@ public class ArbitraryTransaction extends Transaction {
if (arbitraryTransactionData.getVersion() >= 5) { if (arbitraryTransactionData.getVersion() >= 5) {
// Check reported length of the raw data // Check reported length of the raw data
// We should not download the raw data, so validation of that will be performed later // We should not download the raw data, so validation of that will be performed later
if (arbitraryTransactionData.getSize() > DataFile.MAX_FILE_SIZE) { if (arbitraryTransactionData.getSize() > ArbitraryDataFile.MAX_FILE_SIZE) {
return ValidationResult.INVALID_DATA_LENGTH; return ValidationResult.INVALID_DATA_LENGTH;
} }
} }

58
src/test/java/org/qortal/test/DataTests.java
View File

@ -3,7 +3,7 @@ package org.qortal.test;
import org.junit.Before; import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.storage.DataFile; import org.qortal.storage.ArbitraryDataFile;
import org.qortal.test.common.Common; import org.qortal.test.common.Common;
import java.util.Random; import java.util.Random;
@ -20,28 +20,28 @@ public class DataTests extends Common {
@Test @Test
public void testSplitAndJoin() { public void testSplitAndJoin() {
String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890"; String dummyDataString = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
DataFile dataFile = new DataFile(dummyDataString.getBytes()); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(dummyDataString.getBytes());
assertTrue(dataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(62, dataFile.size()); assertEquals(62, arbitraryDataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
// Split into 7 chunks, each 10 bytes long // Split into 7 chunks, each 10 bytes long
dataFile.split(10); arbitraryDataFile.split(10);
assertEquals(7, dataFile.chunkCount()); assertEquals(7, arbitraryDataFile.chunkCount());
// Delete the original file // Delete the original file
dataFile.delete(); arbitraryDataFile.delete();
assertFalse(dataFile.exists()); assertFalse(arbitraryDataFile.exists());
assertEquals(0, dataFile.size()); assertEquals(0, arbitraryDataFile.size());
// Now rebuild the original file from the chunks // Now rebuild the original file from the chunks
assertEquals(7, dataFile.chunkCount()); assertEquals(7, arbitraryDataFile.chunkCount());
dataFile.join(); arbitraryDataFile.join();
// Validate that the original file is intact // Validate that the original file is intact
assertTrue(dataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(62, dataFile.size()); assertEquals(62, arbitraryDataFile.size());
assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", dataFile.digest58()); assertEquals("3eyjYjturyVe61grRX42bprGr3Cvw6ehTy4iknVnosDj", arbitraryDataFile.digest58());
} }
@Test @Test
@ -50,28 +50,28 @@ public class DataTests extends Common {
byte[] randomData = new byte[fileSize]; byte[] randomData = new byte[fileSize];
new Random().nextBytes(randomData); // No need for SecureRandom here new Random().nextBytes(randomData); // No need for SecureRandom here
DataFile dataFile = new DataFile(randomData); ArbitraryDataFile arbitraryDataFile = new ArbitraryDataFile(randomData);
assertTrue(dataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(fileSize, dataFile.size()); assertEquals(fileSize, arbitraryDataFile.size());
String originalFileDigest = dataFile.digest58(); String originalFileDigest = arbitraryDataFile.digest58();
// Split into chunks using 1MiB chunk size // Split into chunks using 1MiB chunk size
dataFile.split(1 * 1024 * 1024); arbitraryDataFile.split(1 * 1024 * 1024);
assertEquals(6, dataFile.chunkCount()); assertEquals(6, arbitraryDataFile.chunkCount());
// Delete the original file // Delete the original file
dataFile.delete(); arbitraryDataFile.delete();
assertFalse(dataFile.exists()); assertFalse(arbitraryDataFile.exists());
assertEquals(0, dataFile.size()); assertEquals(0, arbitraryDataFile.size());
// Now rebuild the original file from the chunks // Now rebuild the original file from the chunks
assertEquals(6, dataFile.chunkCount()); assertEquals(6, arbitraryDataFile.chunkCount());
dataFile.join(); arbitraryDataFile.join();
// Validate that the original file is intact // Validate that the original file is intact
assertTrue(dataFile.exists()); assertTrue(arbitraryDataFile.exists());
assertEquals(fileSize, dataFile.size()); assertEquals(fileSize, arbitraryDataFile.size());
assertEquals(originalFileDigest, dataFile.digest58()); assertEquals(originalFileDigest, arbitraryDataFile.digest58());
} }
} }