Major upgrade of arbitrary transactions

- Adds support for files up 500MiB per transaction (at 2MiB chunk sizes). Previously, the max data size was 4000 bytes.
- Adds a nonce, giving us the option to remove the transaction fees altogether on the data chain.

These features become enabled in version 5 of arbitrary transactions.

src/main/java/org/qortal/data/transaction/ArbitraryTransactionData.java

@@ -32,10 +32,14 @@ public class ArbitraryTransactionData extends TransactionData {
 	private byte[] senderPublicKey;
 
 	private int service;
+	private int nonce;
+	private int size;
 
 	@Schema(example = "raw_data_in_base58")
 	private byte[] data;
 	private DataType dataType;
+	@Schema(example = "chunk_hashes_in_base58")
+	private byte[] chunkHashes;
 	private List<PaymentData> payments;
 
 	// Constructors
@@ -50,14 +54,18 @@ public class ArbitraryTransactionData extends TransactionData {
 	}
 
 	public ArbitraryTransactionData(BaseTransactionData baseTransactionData,
-			int version, int service, byte[] data, DataType dataType, List<PaymentData> payments) {
+			int version, int service, int nonce, int size, byte[] data,
+			DataType dataType, byte[] chunkHashes, List<PaymentData> payments) {
 		super(TransactionType.ARBITRARY, baseTransactionData);
 
 		this.senderPublicKey = baseTransactionData.creatorPublicKey;
 		this.version = version;
 		this.service = service;
+		this.nonce = nonce;
+		this.size = size;
 		this.data = data;
 		this.dataType = dataType;
+		this.chunkHashes = chunkHashes;
 		this.payments = payments;
 	}
 
@@ -75,6 +83,18 @@ public class ArbitraryTransactionData extends TransactionData {
 		return this.service;
 	}
 
+	public int getNonce() {
+		return this.nonce;
+	}
+
+	public void setNonce(int nonce) {
+		this.nonce = nonce;
+	}
+
+	public int getSize() {
+		return this.size;
+	}
+
 	public byte[] getData() {
 		return this.data;
 	}
@@ -91,6 +111,14 @@ public class ArbitraryTransactionData extends TransactionData {
 		this.dataType = dataType;
 	}
 
+	public byte[] getChunkHashes() {
+		return this.chunkHashes;
+	}
+
+	public void setChunkHashes(byte[] chunkHashes) {
+		this.chunkHashes = chunkHashes;
+	}
+
 	public List<PaymentData> getPayments() {
 		return this.payments;
 	}

src/main/java/org/qortal/repository/hsqldb/HSQLDBArbitraryRepository.java

@@ -1,21 +1,14 @@
 package org.qortal.repository.hsqldb;
 
-import java.io.File;
+import org.apache.logging.log4j.LogManager;
-import java.io.IOException;
+import org.apache.logging.log4j.Logger;
-import java.io.OutputStream;
-import java.nio.file.DirectoryNotEmptyException;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-
 import org.qortal.crypto.Crypto;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.data.transaction.ArbitraryTransactionData.DataType;
 import org.qortal.repository.ArbitraryRepository;
 import org.qortal.repository.DataException;
-import org.qortal.settings.Settings;
+import org.qortal.storage.DataFile;
-import org.qortal.utils.Base58;
 
 public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
@@ -23,36 +16,12 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 
 	protected HSQLDBRepository repository;
 
+	private static final Logger LOGGER = LogManager.getLogger(ArbitraryRepository.class);
+
 	public HSQLDBArbitraryRepository(HSQLDBRepository repository) {
 		this.repository = repository;
 	}
 
-	/**
-	 * Returns pathname for saving arbitrary transaction data payloads.
-	 * <p>
-	 * Format: <tt>arbitrary/<sender>/<service><tx-sig>.raw</tt>
-	 * 
-	 * @param arbitraryTransactionData
-	 * @return
-	 */
-	public static String buildPathname(ArbitraryTransactionData arbitraryTransactionData) {
-		String senderAddress = Crypto.toAddress(arbitraryTransactionData.getSenderPublicKey());
-
-		StringBuilder stringBuilder = new StringBuilder(1024);
-
-		stringBuilder.append(Settings.getInstance().getUserPath());
-		stringBuilder.append("arbitrary");
-		stringBuilder.append(File.separator);
-		stringBuilder.append(senderAddress);
-		stringBuilder.append(File.separator);
-		stringBuilder.append(arbitraryTransactionData.getService());
-		stringBuilder.append(File.separator);
-		stringBuilder.append(Base58.encode(arbitraryTransactionData.getSignature()));
-		stringBuilder.append(".raw");
-
-		return stringBuilder.toString();
-	}
-
 	private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException {
 		TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature);
 		if (transactionData == null)
@@ -64,48 +33,89 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 	@Override
 	public boolean isDataLocal(byte[] signature) throws DataException {
 		ArbitraryTransactionData transactionData = getTransactionData(signature);
-		if (transactionData == null)
+		if (transactionData == null) {
 			return false;
+		}
 
 		// Raw data is always available
-		if (transactionData.getDataType() == DataType.RAW_DATA)
+		if (transactionData.getDataType() == DataType.RAW_DATA) {
 			return true;
+		}
 
-		String dataPathname = buildPathname(transactionData);
+		// Load hashes
+		byte[] digest = transactionData.getData();
+		byte[] chunkHashes = transactionData.getChunkHashes();
 
-		Path dataPath = Paths.get(dataPathname);
+		// Load data file(s)
-		return Files.exists(dataPath);
+		DataFile dataFile = DataFile.fromDigest(digest);
+		if (chunkHashes.length > 0) {
+			dataFile.addChunkHashes(chunkHashes);
+		}
+
+		// Check if we already have the complete data file
+		if (dataFile.exists()) {
+			return true;
+		}
+
+		// Alternatively, if we have all the chunks, then it's safe to assume the data is local
+		if (dataFile.allChunksExist(chunkHashes)) {
+			return true;
+		}
+
+		return false;
 	}
 
 	@Override
 	public byte[] fetchData(byte[] signature) throws DataException {
 		ArbitraryTransactionData transactionData = getTransactionData(signature);
-		if (transactionData == null)
+		if (transactionData == null) {
-			return null;
-
-		// Raw data is always available
-		if (transactionData.getDataType() == DataType.RAW_DATA)
-			return transactionData.getData();
-
-		String dataPathname = buildPathname(transactionData);
-
-		Path dataPath = Paths.get(dataPathname);
-		try {
-			return Files.readAllBytes(dataPath);
-		} catch (IOException e) {
 			return null;
 		}
+
+		// Raw data is always available
+		if (transactionData.getDataType() == DataType.RAW_DATA) {
+			return transactionData.getData();
+		}
+
+		// Load hashes
+		byte[] digest = transactionData.getData();
+		byte[] chunkHashes = transactionData.getChunkHashes();
+
+		// Load data file(s)
+		DataFile dataFile = DataFile.fromDigest(digest);
+		if (chunkHashes.length > 0) {
+			dataFile.addChunkHashes(chunkHashes);
+		}
+
+		// If we have the complete data file, return it
+		if (dataFile.exists()) {
+			return dataFile.getBytes();
+		}
+
+		// Alternatively, if we have all the chunks, combine them into a single file
+		if (dataFile.allChunksExist(chunkHashes)) {
+			dataFile.join();
+
+			// Verify that the combined hash matches the expected hash
+			if (digest.equals(dataFile.digest())) {
+				return dataFile.getBytes();
+			}
+		}
+
+		return null;
 	}
 
 	@Override
 	public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
 		// Already hashed? Nothing to do
-		if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH)
+		if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) {
 			return;
+		}
 
 		// Trivial-sized payloads can remain in raw form
-		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE)
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) {
 			return;
+		}
 
 		// Store non-trivial payloads in filesystem and convert transaction's data to hash form
 		byte[] rawData = arbitraryTransactionData.getData();
@@ -115,48 +125,55 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
 		arbitraryTransactionData.setData(dataHash);
 		arbitraryTransactionData.setDataType(DataType.DATA_HASH);
 
-		String dataPathname = buildPathname(arbitraryTransactionData);
+		// Create DataFile
+		DataFile dataFile = new DataFile(rawData);
 
-		Path dataPath = Paths.get(dataPathname);
+		// Verify that the data file is valid, and that it matches the expected hash
-
+		DataFile.ValidationResult validationResult = dataFile.isValid();
-		// Make sure directory structure exists
+		if (validationResult != DataFile.ValidationResult.OK) {
-		try {
+			dataFile.deleteAll();
-			Files.createDirectories(dataPath.getParent());
+			throw new DataException("Invalid data file when attempting to store arbitrary transaction data");
-		} catch (IOException e) {
+		}
-			throw new DataException("Unable to create arbitrary transaction directory", e);
+		if (!dataHash.equals(dataFile.digest())) {
+			dataFile.deleteAll();
+			throw new DataException("Could not verify hash when attempting to store arbitrary transaction data");
 		}
 
-		// Output actual transaction data
+		// Now create chunks if needed
-		try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
+		int chunkCount = dataFile.split(DataFile.CHUNK_SIZE);
-			dataOut.write(rawData);
+		if (chunkCount > 0) {
-		} catch (IOException e) {
+			LOGGER.info(String.format("Successfully split into %d chunk%s:", chunkCount, (chunkCount == 1 ? "" : "s")));
-			throw new DataException("Unable to store arbitrary transaction data", e);
+			LOGGER.info("{}", dataFile.printChunks());
+
+			// Verify that the chunk hashes match those in the transaction
+			byte[] chunkHashes = dataFile.chunkHashes();
+			if (!chunkHashes.equals(arbitraryTransactionData.getChunkHashes())) {
+				dataFile.deleteAll();
+				throw new DataException("Could not verify chunk hashes when attempting to store arbitrary transaction data");
+			}
+
 		}
 	}
 
 	@Override
 	public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
 		// No need to do anything if we still only have raw data, and hence nothing saved in filesystem
-		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
+		if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) {
 			return;
-
-		String dataPathname = buildPathname(arbitraryTransactionData);
-		Path dataPath = Paths.get(dataPathname);
-		try {
-			Files.deleteIfExists(dataPath);
-
-			// Also attempt to delete parent <service> directory if empty
-			Path servicePath = dataPath.getParent();
-			Files.deleteIfExists(servicePath);
-
-			// Also attempt to delete parent <sender's address> directory if empty
-			Path senderpath = servicePath.getParent();
-			Files.deleteIfExists(senderpath);
-		} catch (DirectoryNotEmptyException e) {
-			// One of the parent service/sender directories still has data from other transactions - this is OK
-		} catch (IOException e) {
-			throw new DataException("Unable to delete arbitrary transaction data", e);
 		}
+
+		// Load hashes
+		byte[] digest = arbitraryTransactionData.getData();
+		byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
+
+		// Load data file(s)
+		DataFile dataFile = DataFile.fromDigest(digest);
+		if (chunkHashes.length > 0) {
+			dataFile.addChunkHashes(chunkHashes);
+		}
+
+		// Delete file and chunks
+		dataFile.deleteAll();
 	}
 
 }

src/main/java/org/qortal/repository/hsqldb/HSQLDBDatabaseUpdates.java

@@ -770,6 +770,20 @@ public class HSQLDBDatabaseUpdates {
 							+ "signature Signature, nonce INT NOT NULL, presence_type INT NOT NULL, "
 							+ "timestamp_signature Signature NOT NULL, " + TRANSACTION_KEYS + ")");
 					break;
+				case 34:
+					// ARBITRARY transaction updates for off-chain data storage
+					stmt.execute("CREATE TYPE ArbitraryDataHashes AS VARBINARY(8000)");
+					// We may want to use a nonce rather than a transaction fee on the data chain
+					stmt.execute("ALTER TABLE ArbitraryTransactions ADD nonce INT NOT NULL DEFAULT 0");
+					// We need to know the total size of the data file(s) associated with each transaction
+					stmt.execute("ALTER TABLE ArbitraryTransactions ADD size INT NOT NULL DEFAULT 0");
+					// Larger data files need to be split into chunks, for easier transmission and greater decentralization
+					stmt.execute("ALTER TABLE ArbitraryTransactions ADD chunk_hashes ArbitraryDataHashes");
+					// For finding data files by hash
+					stmt.execute("CREATE INDEX ArbitraryDataIndex ON ArbitraryTransactions (is_data_raw, data)");
+
+					// TODO: resource ID, compression, layers
+					break;
 
 				default:
 					// nothing to do

src/main/java/org/qortal/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java

@@ -20,21 +20,23 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 	}
 
 	TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException {
-		String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?";
+		String sql = "SELECT version, nonce, service, size, is_data_raw, data, chunk_hashes from ArbitraryTransactions WHERE signature = ?";
 
 		try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) {
 			if (resultSet == null)
 				return null;
 
 			int version = resultSet.getInt(1);
-			int service = resultSet.getInt(2);
+			int nonce = resultSet.getInt(2);
-			boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false
+			int service = resultSet.getInt(3);
+			int size = resultSet.getInt(4);
+			boolean isDataRaw = resultSet.getBoolean(5); // NOT NULL, so no null to false
 			DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
-			byte[] data = resultSet.getBytes(4);
+			byte[] data = resultSet.getBytes(6);
+			byte[] chunkHashes = resultSet.getBytes(7);
 
 			List<PaymentData> payments = this.getPaymentsFromSignature(baseTransactionData.getSignature());
-
+			return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments);
-			return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
 		} catch (SQLException e) {
 			throw new DataException("Unable to fetch arbitrary transaction from repository", e);
 		}
@@ -52,7 +54,9 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
 
 		saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey())
 				.bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService())
-				.bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData());
+				.bind("nonce", arbitraryTransactionData.getNonce()).bind("size", arbitraryTransactionData.getSize())
+				.bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData())
+				.bind("chunk_hashes", arbitraryTransactionData.getChunkHashes());
 
 		try {
 			saveHelper.execute(this.repository);

src/main/java/org/qortal/transaction/ArbitraryTransaction.java

@@ -4,12 +4,19 @@ import java.util.List;
 import java.util.stream.Collectors;
 
 import org.qortal.account.Account;
+import org.qortal.crypto.Crypto;
+import org.qortal.crypto.MemoryPoW;
 import org.qortal.data.PaymentData;
 import org.qortal.data.transaction.ArbitraryTransactionData;
 import org.qortal.data.transaction.TransactionData;
 import org.qortal.payment.Payment;
 import org.qortal.repository.DataException;
 import org.qortal.repository.Repository;
+import org.qortal.storage.DataFile;
+import org.qortal.storage.DataFileChunk;
+import org.qortal.transform.TransformationException;
+import org.qortal.transform.transaction.ArbitraryTransactionTransformer;
+import org.qortal.transform.transaction.TransactionTransformer;
 
 public class ArbitraryTransaction extends Transaction {
 
@@ -18,6 +25,10 @@ public class ArbitraryTransaction extends Transaction {
 
 	// Other useful constants
 	public static final int MAX_DATA_SIZE = 4000;
+	public static final int MAX_CHUNK_HASHES_LENGTH = 8000;
+	public static final int HASH_LENGTH = TransactionTransformer.SHA256_LENGTH;
+	public static final int POW_BUFFER_SIZE = 8 * 1024 * 1024; // bytes
+	public static final int POW_DIFFICULTY = 10; // leading zero bits
 
 	// Constructors
 
@@ -42,20 +53,122 @@ public class ArbitraryTransaction extends Transaction {
 
 	// Processing
 
+	public void computeNonce() throws DataException {
+		byte[] transactionBytes;
+
+		try {
+			transactionBytes = TransactionTransformer.toBytesForSigning(this.transactionData);
+		} catch (TransformationException e) {
+			throw new RuntimeException("Unable to transform transaction to byte array for verification", e);
+		}
+
+		// Clear nonce from transactionBytes
+		ArbitraryTransactionTransformer.clearNonce(transactionBytes);
+
+		int difficulty = POW_DIFFICULTY;
+
+		// Calculate nonce
+		this.arbitraryTransactionData.setNonce(MemoryPoW.compute2(transactionBytes, POW_BUFFER_SIZE, difficulty));
+	}
+
 	@Override
 	public ValidationResult isValid() throws DataException {
-		// Check data length
+		// Check that some data - or a data hash - has been supplied
-		if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE)
+		if (arbitraryTransactionData.getData() == null) {
 			return ValidationResult.INVALID_DATA_LENGTH;
+		}
+
+		// Check data length
+		if (arbitraryTransactionData.getData().length < 1 || arbitraryTransactionData.getData().length > MAX_DATA_SIZE) {
+			return ValidationResult.INVALID_DATA_LENGTH;
+		}
+
+		// Check hashes
+		if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.DATA_HASH) {
+			// Check length of data hash
+			if (arbitraryTransactionData.getData().length != HASH_LENGTH) {
+				return ValidationResult.INVALID_DATA_LENGTH;
+			}
+
+			// Version 5+
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
+
+				// Check maximum length of chunk hashes
+				if (chunkHashes != null && chunkHashes.length > MAX_CHUNK_HASHES_LENGTH) {
+					return ValidationResult.INVALID_DATA_LENGTH;
+				}
+
+				// Check expected length of chunk hashes
+				int chunkCount = arbitraryTransactionData.getSize() / DataFileChunk.CHUNK_SIZE;
+				int expectedChunkHashesSize = (chunkCount > 1) ? chunkCount * HASH_LENGTH : 0;
+				if (chunkHashes == null && expectedChunkHashesSize > 0) {
+					return ValidationResult.INVALID_DATA_LENGTH;
+				}
+				if (chunkHashes.length != expectedChunkHashesSize) {
+					return ValidationResult.INVALID_DATA_LENGTH;
+				}
+			}
+		}
+
+		// Check raw data
+		if (arbitraryTransactionData.getDataType() == ArbitraryTransactionData.DataType.RAW_DATA) {
+			// Version 5+
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				// Check reported length of the raw data
+				// We should not download the raw data, so validation of that will be performed later
+				if (arbitraryTransactionData.getSize() > DataFile.MAX_FILE_SIZE) {
+					return ValidationResult.INVALID_DATA_LENGTH;
+				}
+			}
+		}
 
 		// Wrap and delegate final payment validity checks to Payment class
+		// TODO: we won't be able to do this if we are on the data chain where fees may start as zero
 		return new Payment(this.repository).isValid(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(),
 				arbitraryTransactionData.getFee());
 	}
 
+	@Override
+	public boolean isSignatureValid() {
+		byte[] signature = this.transactionData.getSignature();
+		if (signature == null) {
+			return false;
+		}
+
+		byte[] transactionBytes;
+
+		try {
+			transactionBytes = ArbitraryTransactionTransformer.toBytesForSigning(this.transactionData);
+		} catch (TransformationException e) {
+			throw new RuntimeException("Unable to transform transaction to byte array for verification", e);
+		}
+
+		if (!Crypto.verify(this.transactionData.getCreatorPublicKey(), signature, transactionBytes)) {
+			return false;
+		}
+
+		// Nonce wasn't added until version 5+
+		if (arbitraryTransactionData.getVersion() >= 5) {
+
+			int nonce = arbitraryTransactionData.getNonce();
+
+			// Clear nonce from transactionBytes
+			ArbitraryTransactionTransformer.clearNonce(transactionBytes);
+
+			int difficulty = POW_DIFFICULTY;
+
+			// Check nonce
+			return MemoryPoW.verify2(transactionBytes, POW_BUFFER_SIZE, difficulty, nonce);
+		}
+
+		return true;
+	}
+
 	@Override
 	public ValidationResult isProcessable() throws DataException {
 		// Wrap and delegate final payment processable checks to Payment class
+		// TODO: we won't be able to do this if we are on the data chain where fees may start as zero
 		return new Payment(this.repository).isProcessable(arbitraryTransactionData.getSenderPublicKey(), arbitraryTransactionData.getPayments(),
 				arbitraryTransactionData.getFee());
 	}

src/main/java/org/qortal/transform/transaction/ArbitraryTransactionTransformer.java

@@ -26,11 +26,14 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 	// Property lengths
 	private static final int SERVICE_LENGTH = INT_LENGTH;
+	private static final int NONCE_LENGTH = INT_LENGTH;
 	private static final int DATA_TYPE_LENGTH = BYTE_LENGTH;
 	private static final int DATA_SIZE_LENGTH = INT_LENGTH;
+	private static final int RAW_DATA_SIZE_LENGTH = INT_LENGTH;
+	private static final int CHUNKS_SIZE_LENGTH = INT_LENGTH;
 	private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH;
 
-	private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH;
+	private static final int EXTRAS_LENGTH = SERVICE_LENGTH + NONCE_LENGTH + DATA_TYPE_LENGTH + DATA_SIZE_LENGTH + RAW_DATA_SIZE_LENGTH + CHUNKS_SIZE_LENGTH;
 
 	protected static final TransactionLayout layout;
 
@@ -41,8 +44,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		layout.add("transaction's groupID", TransformationType.INT);
 		layout.add("reference", TransformationType.SIGNATURE);
 		layout.add("sender's public key", TransformationType.PUBLIC_KEY);
-		layout.add("number of payments", TransformationType.INT);
+		layout.add("nonce", TransformationType.INT); // Version 5+
 
+		layout.add("number of payments", TransformationType.INT);
 		layout.add("* recipient", TransformationType.ADDRESS);
 		layout.add("* asset ID of payment", TransformationType.LONG);
 		layout.add("* payment amount", TransformationType.AMOUNT);
@@ -51,6 +55,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		layout.add("is data raw?", TransformationType.BOOLEAN);
 		layout.add("data length", TransformationType.INT);
 		layout.add("data", TransformationType.DATA);
+
+		layout.add("raw data size", TransformationType.INT); // Version 5+
+		layout.add("chunk count", TransformationType.INT); // Version 5+
+		layout.add("chunk hashes", TransformationType.DATA); // Version 5+
+
 		layout.add("fee", TransformationType.AMOUNT);
 		layout.add("signature", TransformationType.SIGNATURE);
 	}
@@ -67,6 +76,11 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 		byte[] senderPublicKey = Serialization.deserializePublicKey(byteBuffer);
 
+		int nonce = 0;
+		if (version >= 5) {
+			nonce = byteBuffer.getInt();
+		}
+
 		// Always return a list of payments, even if empty
 		List<PaymentData> payments = new ArrayList<>();
 		if (version != 1) {
@@ -91,6 +105,18 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		byte[] data = new byte[dataSize];
 		byteBuffer.get(data);
 
+		int size = 0;
+		byte[] chunkHashes = null;
+
+		if (version >= 5) {
+			size = byteBuffer.getInt();
+
+			int chunkHashesSize = byteBuffer.getInt();
+
+			chunkHashes = new byte[chunkHashesSize];
+			byteBuffer.get(chunkHashes);
+		}
+
 		long fee = byteBuffer.getLong();
 
 		byte[] signature = new byte[SIGNATURE_LENGTH];
@@ -98,13 +124,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 		BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature);
 
-		return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
+		return new ArbitraryTransactionData(baseTransactionData, version, service, nonce, size, data, dataType, chunkHashes, payments);
 	}
 
 	public static int getDataLength(TransactionData transactionData) throws TransformationException {
 		ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
 
-		int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length;
+		int dataLength = (arbitraryTransactionData.getData() != null) ? arbitraryTransactionData.getData().length : 0;
+		int chunkHashesLength = (arbitraryTransactionData.getChunkHashes() != null) ? arbitraryTransactionData.getChunkHashes().length : 0;
+
+		int length = getBaseLength(transactionData) + EXTRAS_LENGTH + dataLength + chunkHashesLength;
 
 		// Optional payments
 		length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength();
@@ -120,6 +149,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 			transformCommonBytes(transactionData, bytes);
 
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce()));
+			}
+
 			List<PaymentData> payments = arbitraryTransactionData.getPayments();
 			bytes.write(Ints.toByteArray(payments.size()));
 
@@ -133,6 +166,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 			bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length));
 			bytes.write(arbitraryTransactionData.getData());
 
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize()));
+
+				byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
+				int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0;
+				bytes.write(Ints.toByteArray(chunkHashesLength));
+
+				bytes.write(arbitraryTransactionData.getChunkHashes());
+			}
+
 			bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee()));
 
 			if (arbitraryTransactionData.getSignature() != null)
@@ -159,6 +202,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 
 			transformCommonBytes(arbitraryTransactionData, bytes);
 
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				bytes.write(Ints.toByteArray(arbitraryTransactionData.getNonce()));
+			}
+
 			if (arbitraryTransactionData.getVersion() != 1) {
 				List<PaymentData> payments = arbitraryTransactionData.getPayments();
 				bytes.write(Ints.toByteArray(payments.size()));
@@ -182,6 +229,16 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 					break;
 			}
 
+			if (arbitraryTransactionData.getVersion() >= 5) {
+				bytes.write(Ints.toByteArray(arbitraryTransactionData.getSize()));
+
+				byte[] chunkHashes = arbitraryTransactionData.getChunkHashes();
+				int chunkHashesLength = (chunkHashes != null) ? chunkHashes.length : 0;
+				bytes.write(Ints.toByteArray(chunkHashesLength));
+
+				bytes.write(arbitraryTransactionData.getChunkHashes());
+			}
+
 			bytes.write(Longs.toByteArray(arbitraryTransactionData.getFee()));
 
 			// Never append signature
@@ -192,4 +249,13 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
 		}
 	}
 
+	public static void clearNonce(byte[] transactionBytes) {
+		int nonceIndex = TYPE_LENGTH + TIMESTAMP_LENGTH + GROUPID_LENGTH + REFERENCE_LENGTH + PUBLIC_KEY_LENGTH;
+
+		transactionBytes[nonceIndex++] = (byte) 0;
+		transactionBytes[nonceIndex++] = (byte) 0;
+		transactionBytes[nonceIndex++] = (byte) 0;
+		transactionBytes[nonceIndex++] = (byte) 0;
+	}
+
 }

src/test/java/org/qortal/test/common/transaction/ArbitraryTestTransaction.java

@@ -18,6 +18,9 @@ public class ArbitraryTestTransaction extends TestTransaction {
 	public static TransactionData randomTransaction(Repository repository, PrivateKeyAccount account, boolean wantValid) throws DataException {
 		final int version = 4;
 		final int service = 123;
+		final int nonce = 0; // Version 4 doesn't need a nonce
+		final int size = 0; // Version 4 doesn't need a size
+		final byte[] chunkHashes = null; // Version 4 doesn't use chunk hashes
 
 		byte[] data = new byte[1024];
 		random.nextBytes(data);
@@ -31,7 +34,7 @@ public class ArbitraryTestTransaction extends TestTransaction {
 		List<PaymentData> payments = new ArrayList<>();
 		payments.add(new PaymentData(recipient, assetId, amount));
 
-		return new ArbitraryTransactionData(generateBase(account), version, service, data, dataType, payments);
+		return new ArbitraryTransactionData(generateBase(account), version, service, nonce, size, data, dataType, chunkHashes, payments);
 	}
 
 }