AlphaX/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Issues Pull Requests Projects Releases Wiki Activity

Work on auto-update

Browse Source 
Arbitrary transactions now [de]serialize data-type (raw/hash) for v4+ transactions.
Data type also stored in repository. Very small (<=255 byte) data payloads are also stored directly in HSQLDB.

Added ArbitraryDataManager which looks for hash-only data payloads and possibly requests raw data over network
depending on 'policy' (which currently is "fetch everything").

Added networking support for finding, and transferring, arbitrary data payloads.

Minor optimization to message ID generation in Peer.

Minor optimization in Serialization.serializeSizedString()
This commit is contained in:
catbref 2019-06-07 13:12:24 +01:00
parent 4ced9cc3e2
commit 8dd4745c5c
14 changed files with 508 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
src/main/java/org/qora/controller/ArbitraryDataManager.java Normal file
View File

@ -0,0 +1,91 @@
package org.qora.controller;
import java.util.Arrays;
import java.util.List;
import java.util.Random;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qora.api.resource.TransactionsResource.ConfirmationStatus;
import org.qora.data.transaction.ArbitraryTransactionData;
import org.qora.data.transaction.TransactionData;
import org.qora.repository.DataException;
import org.qora.repository.Repository;
import org.qora.repository.RepositoryManager;
import org.qora.transaction.ArbitraryTransaction;
import org.qora.transaction.Transaction.TransactionType;
public class ArbitraryDataManager extends Thread {
private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class);
private static final List<TransactionType> ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY);
private static ArbitraryDataManager instance;
private volatile boolean isStopping = false;
private ArbitraryDataManager() {
}
public static ArbitraryDataManager getInstance() {
if (instance == null)
instance = new ArbitraryDataManager();
return instance;
}
@Override
public void run() {
Thread.currentThread().setName("Arbitrary Data Manager");
try {
while (!isStopping) {
Thread.sleep(2000);
// Any arbitrary transactions we want to fetch data for?
try (final Repository repository = RepositoryManager.getRepository()) {
List<byte[]> signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true);
if (signatures == null || signatures.isEmpty())
continue;
// Filter out those that already have local data
signatures.removeIf(signature -> hasLocalData(repository, signature));
if (signatures.isEmpty())
continue;
// Pick one at random
final int index = new Random().nextInt(signatures.size());
byte[] signature = signatures.get(index);
Controller.getInstance().fetchArbitraryData(signature);
} catch (DataException e) {
LOGGER.error("Repository issue when fetching arbitrary transaction data", e);
}
}
} catch (InterruptedException e) {
return;
}
}
public void shutdown() {
isStopping = true;
this.interrupt();
}
private boolean hasLocalData(final Repository repository, final byte[] signature) {
try {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData))
return true;
ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData);
return arbitraryTransaction.isDataLocal();
} catch (DataException e) {
LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e);
return true;
}
}
}

2
src/main/java/org/qora/controller/AutoUpdate.java
View File

@ -51,7 +51,7 @@ public class AutoUpdate extends Thread {
private static AutoUpdate instance;
private boolean isStopping = false;
private volatile boolean isStopping = false;
private AutoUpdate() {
}

191
src/main/java/org/qora/controller/Controller.java
View File

@ -9,8 +9,12 @@ import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Random;
import java.util.concurrent.locks.ReentrantLock;
import java.util.function.Predicate;
@ -23,15 +27,20 @@ import org.qora.block.Block;
import org.qora.block.BlockChain;
import org.qora.block.BlockGenerator;
import org.qora.controller.Synchronizer.SynchronizationResult;
import org.qora.crypto.Crypto;
import org.qora.data.block.BlockData;
import org.qora.data.network.BlockSummaryData;
import org.qora.data.network.PeerData;
import org.qora.data.transaction.ArbitraryTransactionData;
import org.qora.data.transaction.ArbitraryTransactionData.DataType;
import org.qora.data.transaction.TransactionData;
import org.qora.gui.Gui;
import org.qora.network.Network;
import org.qora.network.Peer;
import org.qora.network.message.ArbitraryDataMessage;
import org.qora.network.message.BlockMessage;
import org.qora.network.message.BlockSummariesMessage;
import org.qora.network.message.GetArbitraryDataMessage;
import org.qora.network.message.GetBlockMessage;
import org.qora.network.message.GetBlockSummariesMessage;
import org.qora.network.message.GetPeersMessage;
@ -51,11 +60,14 @@ import org.qora.repository.RepositoryFactory;
import org.qora.repository.RepositoryManager;
import org.qora.repository.hsqldb.HSQLDBRepositoryFactory;
import org.qora.settings.Settings;
import org.qora.transaction.ArbitraryTransaction;
import org.qora.transaction.Transaction;
import org.qora.transaction.Transaction.TransactionType;
import org.qora.transaction.Transaction.ValidationResult;
import org.qora.ui.UiService;
import org.qora.utils.Base58;
import org.qora.utils.NTP;
import org.qora.utils.Triple;
public class Controller extends Thread {
@ -72,14 +84,35 @@ public class Controller extends Thread {
private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks
private static final Object shutdownLock = new Object();
private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true";
private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000; // ms
private static volatile boolean isStopping = false;
private static BlockGenerator blockGenerator = null;
private static volatile boolean requestSync = false;
private static Controller instance;
private final String buildVersion;
private final long buildTimestamp; // seconds
/**
* Map of recent requests for ARBITRARY transaction data payloads.
* <p>
* Key is original request's message ID<br>
* Value is Triple&lt;transaction signature in base58, first requesting peer, first request's timestamp&gt;
* <p>
* If peer is null then either:<br>
* <ul>
* <li>we are the original requesting peer</li>
* <li>we have already sent data payload to original requesting peer.</li>
* </ul>
* If signature is null then we have already received the data payload and either:<br>
* <ul>
* <li>we are the original requesting peer and have saved it locally</li>
* <li>we have forwarded the data payload (and maybe also saved it locally)</li>
* </ul>
*/
private Map<Integer, Triple<String, Peer, Long>> arbitraryDataRequests = Collections.synchronizedMap(new HashMap<>());
/** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly generated block. */
private final ReentrantLock blockchainLock = new ReentrantLock();
@ -223,6 +256,10 @@ public class Controller extends Thread {
LOGGER.info("Starting controller");
Controller.getInstance().start();
// Arbitrary transaction data manager
LOGGER.info("Starting arbitrary-transaction data manager");
ArbitraryDataManager.getInstance().start();
// Auto-update service
LOGGER.info("Starting auto-update");
AutoUpdate.getInstance().start();
@ -263,6 +300,10 @@ public class Controller extends Thread {
requestSync = false;
potentiallySynchronize();
}
// Clean up arbitrary data request cache
final long requestMinimumTimestamp = NTP.getTime() - ARBITRARY_REQUEST_TIMEOUT;
arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp);
}
}
@ -351,6 +392,10 @@ public class Controller extends Thread {
LOGGER.info("Shutting down auto-update");
AutoUpdate.getInstance().shutdown();
// Arbitrary transaction data manager
LOGGER.info("Shutting down arbitrary-transaction data manager");
ArbitraryDataManager.getInstance().shutdown();
LOGGER.info("Shutting down controller");
this.interrupt();
try {
@ -780,6 +825,107 @@ public class Controller extends Thread {
break;
}
case GET_ARBITRARY_DATA: {
GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message;
byte[] signature = getArbitraryDataMessage.getSignature();
String signature58 = Base58.encode(signature);
Long timestamp = NTP.getTime();
Triple<String, Peer, Long> newEntry = new Triple<>(signature58, peer, timestamp);
// If we've seen this request recently, then ignore
if (arbitraryDataRequests.putIfAbsent(message.getId(), newEntry) != null)
break;
// Do we even have this transaction?
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY)
break;
ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData);
// If we have the data then send it
if (transaction.isDataLocal()) {
byte[] data = transaction.fetchData();
if (data == null)
break;
// Update requests map to reflect that we've sent it
newEntry = new Triple<>(signature58, null, timestamp);
arbitraryDataRequests.put(message.getId(), newEntry);
Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data);
arbitraryDataMessage.setId(message.getId());
if (!peer.sendMessage(arbitraryDataMessage))
peer.disconnect("failed to send arbitrary data");
break;
}
// Ask our other peers if they have it
Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e);
}
break;
}
case ARBITRARY_DATA: {
ArbitraryDataMessage arbitraryDataMessage = (ArbitraryDataMessage) message;
// Do we have a pending request for this data?
Triple<String, Peer, Long> request = arbitraryDataRequests.get(message.getId());
if (request == null || request.getA() == null)
break;
// Does this message's signature match what we're expecting?
byte[] signature = arbitraryDataMessage.getSignature();
String signature58 = Base58.encode(signature);
if (!request.getA().equals(signature58))
break;
byte[] data = arbitraryDataMessage.getData();
// Check transaction exists and payload hash is correct
try (final Repository repository = RepositoryManager.getRepository()) {
TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData))
break;
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
byte[] actualHash = Crypto.digest(data);
// "data" from repository will always be hash of actual raw data
if (!Arrays.equals(arbitraryTransactionData.getData(), actualHash))
break;
// Update requests map to reflect that we've received it
Triple<String, Peer, Long> newEntry = new Triple<>(null, null, request.getC());
arbitraryDataRequests.put(message.getId(), newEntry);
// Save payload locally
// TODO: storage policy
arbitraryTransactionData.setDataType(DataType.RAW_DATA);
arbitraryTransactionData.setData(data);
repository.getArbitraryRepository().save(arbitraryTransactionData);
repository.saveChanges();
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e);
}
Peer requestingPeer = request.getB();
if (requestingPeer != null) {
// Forward to requesting peer;
if (!requestingPeer.sendMessage(arbitraryDataMessage))
requestingPeer.disconnect("failed to forward arbitrary data");
}
break;
}
default:
LOGGER.debug(String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer));
break;
@ -788,6 +934,51 @@ public class Controller extends Thread {
// Utilities
public byte[] fetchArbitraryData(byte[] signature) throws InterruptedException {
// Build request
Message getArbitraryDataMessage = new GetArbitraryDataMessage(signature);
// Save our request into requests map
String signature58 = Base58.encode(signature);
Triple<String, Peer, Long> requestEntry = new Triple<>(signature58, null, NTP.getTime());
// Assign random ID to this message
int id;
do {
id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
// Put queue into map (keyed by message ID) so we can poll for a response
// If putIfAbsent() doesn't return null, then this ID is already taken
} while (arbitraryDataRequests.put(id, requestEntry) != null);
getArbitraryDataMessage.setId(id);
// Broadcast request
Network.getInstance().broadcast(peer -> peer.getVersion() < 2 ? null : getArbitraryDataMessage);
// Poll to see if data has arrived
final long singleWait = 100;
long totalWait = 0;
while (totalWait < ARBITRARY_REQUEST_TIMEOUT) {
Thread.sleep(singleWait);
requestEntry = arbitraryDataRequests.get(id);
if (requestEntry == null)
return null;
if (requestEntry.getA() == null)
break;
totalWait += singleWait;
}
try (final Repository repository = RepositoryManager.getRepository()) {
return repository.getArbitraryRepository().fetchData(signature);
} catch (DataException e) {
LOGGER.error(String.format("Repository issue while fetching arbitrary transaction data"), e);
return null;
}
}
public static final Predicate<Peer> hasPeerMisbehaved = peer -> {
Long lastMisbehaved = peer.getPeerData().getLastMisbehaved();
return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF;

2
src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java
View File

@ -11,7 +11,6 @@ import org.qora.data.PaymentData;
import org.qora.transaction.Transaction.TransactionType;
import io.swagger.v3.oas.annotations.media.Schema;
import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
// All properties to be converted to JSON via JAXB
@XmlAccessorType(XmlAccessType.FIELD)
@ -21,7 +20,6 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode;
public class ArbitraryTransactionData extends TransactionData {
// "data" field types
@Schema(accessMode = AccessMode.READ_ONLY)
public enum DataType {
RAW_DATA,
DATA_HASH;

4
src/main/java/org/qora/network/Peer.java
View File

@ -357,12 +357,12 @@ public class Peer implements Runnable {
// Assign random ID to this message
int id;
do {
id = new SecureRandom().nextInt(Integer.MAX_VALUE - 1) + 1;
message.setId(id);
id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1;
// Put queue into map (keyed by message ID) so we can poll for a response
// If putIfAbsent() doesn't return null, then this ID is already taken
} while (this.replyQueues.putIfAbsent(id, blockingQueue) != null);
message.setId(id);
// Try to send message
if (!this.sendMessage(message)) {

73
src/main/java/org/qora/network/message/ArbitraryDataMessage.java Normal file
View File

@ -0,0 +1,73 @@
package org.qora.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import org.qora.transform.Transformer;
import com.google.common.primitives.Ints;
public class ArbitraryDataMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private byte[] signature;
private byte[] data;
public ArbitraryDataMessage(byte[] signature, byte[] data) {
this(-1, signature, data);
}
private ArbitraryDataMessage(int id, byte[] signature, byte[] data) {
super(id, MessageType.ARBITRARY_DATA);
this.signature = signature;
this.data = data;
}
public byte[] getSignature() {
return this.signature;
}
public byte[] getData() {
return this.data;
}
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException {
byte[] signature = new byte[SIGNATURE_LENGTH];
byteBuffer.get(signature);
int dataLength = byteBuffer.getInt();
if (byteBuffer.remaining() != dataLength)
return null;
byte[] data = new byte[dataLength];
byteBuffer.get(data);
return new ArbitraryDataMessage(id, signature, data);
}
@Override
protected byte[] toData() {
if (this.data == null)
return null;
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
bytes.write(this.signature);
bytes.write(Ints.toByteArray(this.data.length));
bytes.write(this.data);
return bytes.toByteArray();
} catch (IOException e) {
return null;
}
}
}

54
src/main/java/org/qora/network/message/GetArbitraryDataMessage.java Normal file
View File

@ -0,0 +1,54 @@
package org.qora.network.message;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import org.qora.transform.Transformer;
public class GetArbitraryDataMessage extends Message {
private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH;
private byte[] signature;
public GetArbitraryDataMessage(byte[] signature) {
this(-1, signature);
}
private GetArbitraryDataMessage(int id, byte[] signature) {
super(id, MessageType.GET_ARBITRARY_DATA);
this.signature = signature;
}
public byte[] getSignature() {
return this.signature;
}
public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException {
if (bytes.remaining() != SIGNATURE_LENGTH)
return null;
byte[] signature = new byte[SIGNATURE_LENGTH];
bytes.get(signature);
return new GetArbitraryDataMessage(id, signature);
}
@Override
protected byte[] toData() {
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
bytes.write(this.signature);
return bytes.toByteArray();
} catch (IOException e) {
return null;
}
}
}

4
src/main/java/org/qora/network/message/Message.java
View File

@ -68,7 +68,9 @@ public abstract class Message {
HEIGHT_V2(19),
GET_TRANSACTION(20),
GET_UNCONFIRMED_TRANSACTIONS(21),
TRANSACTION_SIGNATURES(22);
TRANSACTION_SIGNATURES(22),
GET_ARBITRARY_DATA(23),
ARBITRARY_DATA(24);
public final int value;
public final Method fromByteBuffer;

78
src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java
View File

@ -19,6 +19,8 @@ import org.qora.utils.Base58;
public class HSQLDBArbitraryRepository implements ArbitraryRepository {
private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY
protected HSQLDBRepository repository;
public HSQLDBArbitraryRepository(HSQLDBRepository repository) {
@ -51,30 +53,42 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
return stringBuilder.toString();
}
private String buildPathname(byte[] signature) throws DataException {
private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException {
TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature);
if (transactionData == null)
return null;
return buildPathname((ArbitraryTransactionData) transactionData);
return (ArbitraryTransactionData) transactionData;
}
@Override
public boolean isDataLocal(byte[] signature) throws DataException {
String dataPathname = buildPathname(signature);
if (dataPathname == null)
ArbitraryTransactionData transactionData = getTransactionData(signature);
if (transactionData == null)
return false;
// Raw data is always available
if (transactionData.getDataType() == DataType.RAW_DATA)
return true;
String dataPathname = buildPathname(transactionData);
Path dataPath = Paths.get(dataPathname);
return Files.exists(dataPath);
}
@Override
public byte[] fetchData(byte[] signature) throws DataException {
String dataPathname = buildPathname(signature);
if (dataPathname == null)
ArbitraryTransactionData transactionData = getTransactionData(signature);
if (transactionData == null)
return null;
// Raw data is always available
if (transactionData.getDataType() == DataType.RAW_DATA)
return transactionData.getData();
String dataPathname = buildPathname(transactionData);
Path dataPath = Paths.get(dataPathname);
try {
return Files.readAllBytes(dataPath);
@ -85,37 +99,47 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository {
@Override
public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
// Refuse to store raw data in the repository - it needs to be saved elsewhere!
if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) {
byte[] rawData = arbitraryTransactionData.getData();
// Already hashed? Nothing to do
if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH)
return;
// Calculate hash of data and update our transaction to use that
byte[] dataHash = Crypto.digest(rawData);
arbitraryTransactionData.setData(dataHash);
arbitraryTransactionData.setDataType(DataType.DATA_HASH);
// Trivial-sized payloads can remain in raw form
if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE)
return;
String dataPathname = buildPathname(arbitraryTransactionData);
// Store non-trivial payloads in filesystem and convert transaction's data to hash form
byte[] rawData = arbitraryTransactionData.getData();
Path dataPath = Paths.get(dataPathname);
// Calculate hash of data and update our transaction to use that
byte[] dataHash = Crypto.digest(rawData);
arbitraryTransactionData.setData(dataHash);
arbitraryTransactionData.setDataType(DataType.DATA_HASH);
// Make sure directory structure exists
try {
Files.createDirectories(dataPath.getParent());
} catch (IOException e) {
throw new DataException("Unable to create arbitrary transaction directory", e);
}
String dataPathname = buildPathname(arbitraryTransactionData);
// Output actual transaction data
try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
dataOut.write(rawData);
} catch (IOException e) {
throw new DataException("Unable to store arbitrary transaction data", e);
}
Path dataPath = Paths.get(dataPathname);
// Make sure directory structure exists
try {
Files.createDirectories(dataPath.getParent());
} catch (IOException e) {
throw new DataException("Unable to create arbitrary transaction directory", e);
}
// Output actual transaction data
try (OutputStream dataOut = Files.newOutputStream(dataPath)) {
dataOut.write(rawData);
} catch (IOException e) {
throw new DataException("Unable to store arbitrary transaction data", e);
}
}
@Override
public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException {
// No need to do anything if we still only have raw data, and hence nothing saved in filesystem
if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
return;
String dataPathname = buildPathname(arbitraryTransactionData);
Path dataPath = Paths.get(dataPathname);
try {

8
src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java
View File

@ -758,6 +758,14 @@ public class HSQLDBDatabaseUpdates {
stmt.execute("CREATE INDEX TransactionApprovalHeightIndex on Transactions (approval_height)");
break;
case 52:
// Arbitrary transactions changes to allow storage of very small payloads locally
stmt.execute("CREATE TYPE ArbitraryData AS VARBINARY(255)");
stmt.execute("ALTER TABLE ArbitraryTransactions ADD COLUMN is_data_raw BOOLEAN NOT NULL");
stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash ArbitraryData");
stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash RENAME TO data");
break;
default:
// nothing to do
return false;

14
src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java
View File

@ -20,7 +20,7 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
}
TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException {
final String sql = "SELECT version, service, data_hash from ArbitraryTransactions WHERE signature = ?";
final String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?";
try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) {
if (resultSet == null)
@ -28,11 +28,13 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
int version = resultSet.getInt(1);
int service = resultSet.getInt(2);
byte[] dataHash = resultSet.getBytes(3);
boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false
DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH;
byte[] data = resultSet.getBytes(4);
List<PaymentData> payments = this.getPaymentsFromSignature(baseTransactionData.getSignature());
return new ArbitraryTransactionData(baseTransactionData, version, service, dataHash, DataType.DATA_HASH, payments);
return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
} catch (SQLException e) {
throw new DataException("Unable to fetch arbitrary transaction from repository", e);
}
@ -42,15 +44,15 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos
public void save(TransactionData transactionData) throws DataException {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
// Refuse to store raw data in the repository - it needs to be saved elsewhere!
if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA)
// For V4+, we might not store raw data in the repository but elsewhere
if (arbitraryTransactionData.getVersion() >= 4)
this.repository.getArbitraryRepository().save(arbitraryTransactionData);
HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions");
saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey())
.bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService())
.bind("data_hash", arbitraryTransactionData.getData());
.bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData());
try {
saveHelper.execute(this.repository);

1
src/main/java/org/qora/transaction/ArbitraryTransaction.java
View File

@ -148,6 +148,7 @@ public class ArbitraryTransaction extends Transaction {
return this.repository.getArbitraryRepository().isDataLocal(this.transactionData.getSignature());
}
/** Returns arbitrary data payload, fetching from network if needed. Can block for a while! */
public byte[] fetchData() throws DataException {
// If local, read from file
if (isDataLocal())

27
src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java
View File

@ -27,7 +27,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
// Property lengths
private static final int SERVICE_LENGTH = INT_LENGTH;
private static final int DATA_TYPE_LENGTH = BYTE_LENGTH;
private static final int DATA_SIZE_LENGTH = INT_LENGTH;
private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH;
private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_SIZE_LENGTH;
@ -47,6 +49,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
layout.add("* payment amount", TransformationType.AMOUNT);
layout.add("service ID", TransformationType.INT);
layout.add("is data raw?", TransformationType.BOOLEAN);
layout.add("data length", TransformationType.INT);
layout.add("data", TransformationType.DATA);
layout.add("fee", TransformationType.AMOUNT);
@ -78,6 +81,15 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
int service = byteBuffer.getInt();
// With V4+ we might be receiving hash of data instead of actual raw data
DataType dataType = DataType.RAW_DATA;
if (version >= 4) {
boolean isRaw = byteBuffer.get() != 0;
if (!isRaw)
dataType = DataType.DATA_HASH;
}
int dataSize = byteBuffer.getInt();
// Don't allow invalid dataSize here to avoid run-time issues
if (dataSize > ArbitraryTransaction.MAX_DATA_SIZE)
@ -93,17 +105,21 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature);
return new ArbitraryTransactionData(baseTransactionData, version, service, data, DataType.RAW_DATA, payments);
return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments);
}
public static int getDataLength(TransactionData transactionData) throws TransformationException {
ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData;
int length = getBaseLength(transactionData) + EXTRAS_LENGTH;
int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length;
// V4+ transactions have data type
if (arbitraryTransactionData.getVersion() >= 4)
length += DATA_TYPE_LENGTH;
// V3+ transactions have optional payments
if (arbitraryTransactionData.getVersion() >= 3)
length += arbitraryTransactionData.getData().length + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength();
length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength();
return length;
}
@ -126,6 +142,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
bytes.write(Ints.toByteArray(arbitraryTransactionData.getService()));
// V4+ also has data type
if (arbitraryTransactionData.getVersion() >= 4)
bytes.write((byte) (arbitraryTransactionData.getDataType() == DataType.RAW_DATA ? 1 : 0));
bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length));
bytes.write(arbitraryTransactionData.getData());
@ -188,6 +208,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer {
bytes.write(Ints.toByteArray(arbitraryTransactionData.getService()));
bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length));
switch (arbitraryTransactionData.getDataType()) {
case DATA_HASH:
bytes.write(arbitraryTransactionData.getData());

2
src/main/java/org/qora/utils/Serialization.java
View File

@ -94,7 +94,7 @@ public class Serialization {
public static void serializeSizedString(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException {
byte[] stringBytes = string.getBytes("UTF-8");
bytes.write(Ints.toByteArray(stringBytes.length));
bytes.write(string.getBytes("UTF-8"));
bytes.write(stringBytes);
}
public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) throws TransformationException {