diff --git a/src/main/java/org/qora/controller/ArbitraryDataManager.java b/src/main/java/org/qora/controller/ArbitraryDataManager.java new file mode 100644 index 00000000..dbc37938 --- /dev/null +++ b/src/main/java/org/qora/controller/ArbitraryDataManager.java @@ -0,0 +1,91 @@ +package org.qora.controller; + +import java.util.Arrays; +import java.util.List; +import java.util.Random; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qora.api.resource.TransactionsResource.ConfirmationStatus; +import org.qora.data.transaction.ArbitraryTransactionData; +import org.qora.data.transaction.TransactionData; +import org.qora.repository.DataException; +import org.qora.repository.Repository; +import org.qora.repository.RepositoryManager; +import org.qora.transaction.ArbitraryTransaction; +import org.qora.transaction.Transaction.TransactionType; + +public class ArbitraryDataManager extends Thread { + + private static final Logger LOGGER = LogManager.getLogger(ArbitraryDataManager.class); + private static final List ARBITRARY_TX_TYPE = Arrays.asList(TransactionType.ARBITRARY); + + private static ArbitraryDataManager instance; + + private volatile boolean isStopping = false; + + private ArbitraryDataManager() { + } + + public static ArbitraryDataManager getInstance() { + if (instance == null) + instance = new ArbitraryDataManager(); + + return instance; + } + + @Override + public void run() { + Thread.currentThread().setName("Arbitrary Data Manager"); + + try { + while (!isStopping) { + Thread.sleep(2000); + + // Any arbitrary transactions we want to fetch data for? + try (final Repository repository = RepositoryManager.getRepository()) { + List signatures = repository.getTransactionRepository().getSignaturesMatchingCriteria(null, null, null, ARBITRARY_TX_TYPE, null, null, ConfirmationStatus.BOTH, null, null, true); + if (signatures == null || signatures.isEmpty()) + continue; + + // Filter out those that already have local data + signatures.removeIf(signature -> hasLocalData(repository, signature)); + + if (signatures.isEmpty()) + continue; + + // Pick one at random + final int index = new Random().nextInt(signatures.size()); + byte[] signature = signatures.get(index); + + Controller.getInstance().fetchArbitraryData(signature); + } catch (DataException e) { + LOGGER.error("Repository issue when fetching arbitrary transaction data", e); + } + } + } catch (InterruptedException e) { + return; + } + } + + public void shutdown() { + isStopping = true; + this.interrupt(); + } + + private boolean hasLocalData(final Repository repository, final byte[] signature) { + try { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData)) + return true; + + ArbitraryTransaction arbitraryTransaction = new ArbitraryTransaction(repository, transactionData); + + return arbitraryTransaction.isDataLocal(); + } catch (DataException e) { + LOGGER.error("Repository issue when checking arbitrary transaction's data is local", e); + return true; + } + } + +} diff --git a/src/main/java/org/qora/controller/AutoUpdate.java b/src/main/java/org/qora/controller/AutoUpdate.java index 7c1795b5..b8c1a1c0 100644 --- a/src/main/java/org/qora/controller/AutoUpdate.java +++ b/src/main/java/org/qora/controller/AutoUpdate.java @@ -51,7 +51,7 @@ public class AutoUpdate extends Thread { private static AutoUpdate instance; - private boolean isStopping = false; + private volatile boolean isStopping = false; private AutoUpdate() { } diff --git a/src/main/java/org/qora/controller/Controller.java b/src/main/java/org/qora/controller/Controller.java index 0fe52006..ecb729cc 100644 --- a/src/main/java/org/qora/controller/Controller.java +++ b/src/main/java/org/qora/controller/Controller.java @@ -9,8 +9,12 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.Random; import java.util.concurrent.locks.ReentrantLock; import java.util.function.Predicate; @@ -23,15 +27,20 @@ import org.qora.block.Block; import org.qora.block.BlockChain; import org.qora.block.BlockGenerator; import org.qora.controller.Synchronizer.SynchronizationResult; +import org.qora.crypto.Crypto; import org.qora.data.block.BlockData; import org.qora.data.network.BlockSummaryData; import org.qora.data.network.PeerData; +import org.qora.data.transaction.ArbitraryTransactionData; +import org.qora.data.transaction.ArbitraryTransactionData.DataType; import org.qora.data.transaction.TransactionData; import org.qora.gui.Gui; import org.qora.network.Network; import org.qora.network.Peer; +import org.qora.network.message.ArbitraryDataMessage; import org.qora.network.message.BlockMessage; import org.qora.network.message.BlockSummariesMessage; +import org.qora.network.message.GetArbitraryDataMessage; import org.qora.network.message.GetBlockMessage; import org.qora.network.message.GetBlockSummariesMessage; import org.qora.network.message.GetPeersMessage; @@ -51,11 +60,14 @@ import org.qora.repository.RepositoryFactory; import org.qora.repository.RepositoryManager; import org.qora.repository.hsqldb.HSQLDBRepositoryFactory; import org.qora.settings.Settings; +import org.qora.transaction.ArbitraryTransaction; import org.qora.transaction.Transaction; +import org.qora.transaction.Transaction.TransactionType; import org.qora.transaction.Transaction.ValidationResult; import org.qora.ui.UiService; import org.qora.utils.Base58; import org.qora.utils.NTP; +import org.qora.utils.Triple; public class Controller extends Thread { @@ -72,14 +84,35 @@ public class Controller extends Thread { private static final int MAX_BLOCKCHAIN_TIP_AGE = 5; // blocks private static final Object shutdownLock = new Object(); private static final String repositoryUrlTemplate = "jdbc:hsqldb:file:%s/blockchain;create=true"; + private static final long ARBITRARY_REQUEST_TIMEOUT = 5 * 1000; // ms private static volatile boolean isStopping = false; private static BlockGenerator blockGenerator = null; private static volatile boolean requestSync = false; private static Controller instance; + private final String buildVersion; private final long buildTimestamp; // seconds + /** + * Map of recent requests for ARBITRARY transaction data payloads. + *

+ * Key is original request's message ID
+ * Value is Triple<transaction signature in base58, first requesting peer, first request's timestamp> + *

+ * If peer is null then either:
+ *

+ * If signature is null then we have already received the data payload and either:
+ * + */ + private Map> arbitraryDataRequests = Collections.synchronizedMap(new HashMap<>()); + /** Lock for only allowing one blockchain-modifying codepath at a time. e.g. synchronization or newly generated block. */ private final ReentrantLock blockchainLock = new ReentrantLock(); @@ -223,6 +256,10 @@ public class Controller extends Thread { LOGGER.info("Starting controller"); Controller.getInstance().start(); + // Arbitrary transaction data manager + LOGGER.info("Starting arbitrary-transaction data manager"); + ArbitraryDataManager.getInstance().start(); + // Auto-update service LOGGER.info("Starting auto-update"); AutoUpdate.getInstance().start(); @@ -263,6 +300,10 @@ public class Controller extends Thread { requestSync = false; potentiallySynchronize(); } + + // Clean up arbitrary data request cache + final long requestMinimumTimestamp = NTP.getTime() - ARBITRARY_REQUEST_TIMEOUT; + arbitraryDataRequests.entrySet().removeIf(entry -> entry.getValue().getC() < requestMinimumTimestamp); } } @@ -351,6 +392,10 @@ public class Controller extends Thread { LOGGER.info("Shutting down auto-update"); AutoUpdate.getInstance().shutdown(); + // Arbitrary transaction data manager + LOGGER.info("Shutting down arbitrary-transaction data manager"); + ArbitraryDataManager.getInstance().shutdown(); + LOGGER.info("Shutting down controller"); this.interrupt(); try { @@ -780,6 +825,107 @@ public class Controller extends Thread { break; } + case GET_ARBITRARY_DATA: { + GetArbitraryDataMessage getArbitraryDataMessage = (GetArbitraryDataMessage) message; + + byte[] signature = getArbitraryDataMessage.getSignature(); + String signature58 = Base58.encode(signature); + Long timestamp = NTP.getTime(); + Triple newEntry = new Triple<>(signature58, peer, timestamp); + + // If we've seen this request recently, then ignore + if (arbitraryDataRequests.putIfAbsent(message.getId(), newEntry) != null) + break; + + // Do we even have this transaction? + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || transactionData.getType() != TransactionType.ARBITRARY) + break; + + ArbitraryTransaction transaction = new ArbitraryTransaction(repository, transactionData); + + // If we have the data then send it + if (transaction.isDataLocal()) { + byte[] data = transaction.fetchData(); + if (data == null) + break; + + // Update requests map to reflect that we've sent it + newEntry = new Triple<>(signature58, null, timestamp); + arbitraryDataRequests.put(message.getId(), newEntry); + + Message arbitraryDataMessage = new ArbitraryDataMessage(signature, data); + arbitraryDataMessage.setId(message.getId()); + if (!peer.sendMessage(arbitraryDataMessage)) + peer.disconnect("failed to send arbitrary data"); + + break; + } + + // Ask our other peers if they have it + Network.getInstance().broadcast(broadcastPeer -> broadcastPeer == peer ? null : message); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); + } + + break; + } + + case ARBITRARY_DATA: { + ArbitraryDataMessage arbitraryDataMessage = (ArbitraryDataMessage) message; + + // Do we have a pending request for this data? + Triple request = arbitraryDataRequests.get(message.getId()); + if (request == null || request.getA() == null) + break; + + // Does this message's signature match what we're expecting? + byte[] signature = arbitraryDataMessage.getSignature(); + String signature58 = Base58.encode(signature); + if (!request.getA().equals(signature58)) + break; + + byte[] data = arbitraryDataMessage.getData(); + + // Check transaction exists and payload hash is correct + try (final Repository repository = RepositoryManager.getRepository()) { + TransactionData transactionData = repository.getTransactionRepository().fromSignature(signature); + if (transactionData == null || !(transactionData instanceof ArbitraryTransactionData)) + break; + + ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; + + byte[] actualHash = Crypto.digest(data); + + // "data" from repository will always be hash of actual raw data + if (!Arrays.equals(arbitraryTransactionData.getData(), actualHash)) + break; + + // Update requests map to reflect that we've received it + Triple newEntry = new Triple<>(null, null, request.getC()); + arbitraryDataRequests.put(message.getId(), newEntry); + + // Save payload locally + // TODO: storage policy + arbitraryTransactionData.setDataType(DataType.RAW_DATA); + arbitraryTransactionData.setData(data); + repository.getArbitraryRepository().save(arbitraryTransactionData); + repository.saveChanges(); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while finding arbitrary transaction data for peer %s", peer), e); + } + + Peer requestingPeer = request.getB(); + if (requestingPeer != null) { + // Forward to requesting peer; + if (!requestingPeer.sendMessage(arbitraryDataMessage)) + requestingPeer.disconnect("failed to forward arbitrary data"); + } + + break; + } + default: LOGGER.debug(String.format("Unhandled %s message [ID %d] from peer %s", message.getType().name(), message.getId(), peer)); break; @@ -788,6 +934,51 @@ public class Controller extends Thread { // Utilities + public byte[] fetchArbitraryData(byte[] signature) throws InterruptedException { + // Build request + Message getArbitraryDataMessage = new GetArbitraryDataMessage(signature); + + // Save our request into requests map + String signature58 = Base58.encode(signature); + Triple requestEntry = new Triple<>(signature58, null, NTP.getTime()); + + // Assign random ID to this message + int id; + do { + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; + + // Put queue into map (keyed by message ID) so we can poll for a response + // If putIfAbsent() doesn't return null, then this ID is already taken + } while (arbitraryDataRequests.put(id, requestEntry) != null); + getArbitraryDataMessage.setId(id); + + // Broadcast request + Network.getInstance().broadcast(peer -> peer.getVersion() < 2 ? null : getArbitraryDataMessage); + + // Poll to see if data has arrived + final long singleWait = 100; + long totalWait = 0; + while (totalWait < ARBITRARY_REQUEST_TIMEOUT) { + Thread.sleep(singleWait); + + requestEntry = arbitraryDataRequests.get(id); + if (requestEntry == null) + return null; + + if (requestEntry.getA() == null) + break; + + totalWait += singleWait; + } + + try (final Repository repository = RepositoryManager.getRepository()) { + return repository.getArbitraryRepository().fetchData(signature); + } catch (DataException e) { + LOGGER.error(String.format("Repository issue while fetching arbitrary transaction data"), e); + return null; + } + } + public static final Predicate hasPeerMisbehaved = peer -> { Long lastMisbehaved = peer.getPeerData().getLastMisbehaved(); return lastMisbehaved != null && lastMisbehaved > NTP.getTime() - MISBEHAVIOUR_COOLOFF; diff --git a/src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java b/src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java index ab72ad03..e2078e59 100644 --- a/src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java +++ b/src/main/java/org/qora/data/transaction/ArbitraryTransactionData.java @@ -11,7 +11,6 @@ import org.qora.data.PaymentData; import org.qora.transaction.Transaction.TransactionType; import io.swagger.v3.oas.annotations.media.Schema; -import io.swagger.v3.oas.annotations.media.Schema.AccessMode; // All properties to be converted to JSON via JAXB @XmlAccessorType(XmlAccessType.FIELD) @@ -21,7 +20,6 @@ import io.swagger.v3.oas.annotations.media.Schema.AccessMode; public class ArbitraryTransactionData extends TransactionData { // "data" field types - @Schema(accessMode = AccessMode.READ_ONLY) public enum DataType { RAW_DATA, DATA_HASH; diff --git a/src/main/java/org/qora/network/Peer.java b/src/main/java/org/qora/network/Peer.java index ea3d798c..3bf5a239 100644 --- a/src/main/java/org/qora/network/Peer.java +++ b/src/main/java/org/qora/network/Peer.java @@ -357,12 +357,12 @@ public class Peer implements Runnable { // Assign random ID to this message int id; do { - id = new SecureRandom().nextInt(Integer.MAX_VALUE - 1) + 1; - message.setId(id); + id = new Random().nextInt(Integer.MAX_VALUE - 1) + 1; // Put queue into map (keyed by message ID) so we can poll for a response // If putIfAbsent() doesn't return null, then this ID is already taken } while (this.replyQueues.putIfAbsent(id, blockingQueue) != null); + message.setId(id); // Try to send message if (!this.sendMessage(message)) { diff --git a/src/main/java/org/qora/network/message/ArbitraryDataMessage.java b/src/main/java/org/qora/network/message/ArbitraryDataMessage.java new file mode 100644 index 00000000..83946cf7 --- /dev/null +++ b/src/main/java/org/qora/network/message/ArbitraryDataMessage.java @@ -0,0 +1,73 @@ +package org.qora.network.message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import org.qora.transform.Transformer; + +import com.google.common.primitives.Ints; + +public class ArbitraryDataMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private byte[] signature; + private byte[] data; + + public ArbitraryDataMessage(byte[] signature, byte[] data) { + this(-1, signature, data); + } + + private ArbitraryDataMessage(int id, byte[] signature, byte[] data) { + super(id, MessageType.ARBITRARY_DATA); + + this.signature = signature; + this.data = data; + } + + public byte[] getSignature() { + return this.signature; + } + + public byte[] getData() { + return this.data; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws UnsupportedEncodingException { + byte[] signature = new byte[SIGNATURE_LENGTH]; + byteBuffer.get(signature); + + int dataLength = byteBuffer.getInt(); + + if (byteBuffer.remaining() != dataLength) + return null; + + byte[] data = new byte[dataLength]; + byteBuffer.get(data); + + return new ArbitraryDataMessage(id, signature, data); + } + + @Override + protected byte[] toData() { + if (this.data == null) + return null; + + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + bytes.write(Ints.toByteArray(this.data.length)); + + bytes.write(this.data); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qora/network/message/GetArbitraryDataMessage.java b/src/main/java/org/qora/network/message/GetArbitraryDataMessage.java new file mode 100644 index 00000000..b67bc9c5 --- /dev/null +++ b/src/main/java/org/qora/network/message/GetArbitraryDataMessage.java @@ -0,0 +1,54 @@ +package org.qora.network.message; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; + +import org.qora.transform.Transformer; + +public class GetArbitraryDataMessage extends Message { + + private static final int SIGNATURE_LENGTH = Transformer.SIGNATURE_LENGTH; + + private byte[] signature; + + public GetArbitraryDataMessage(byte[] signature) { + this(-1, signature); + } + + private GetArbitraryDataMessage(int id, byte[] signature) { + super(id, MessageType.GET_ARBITRARY_DATA); + + this.signature = signature; + } + + public byte[] getSignature() { + return this.signature; + } + + public static Message fromByteBuffer(int id, ByteBuffer bytes) throws UnsupportedEncodingException { + if (bytes.remaining() != SIGNATURE_LENGTH) + return null; + + byte[] signature = new byte[SIGNATURE_LENGTH]; + + bytes.get(signature); + + return new GetArbitraryDataMessage(id, signature); + } + + @Override + protected byte[] toData() { + try { + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + bytes.write(this.signature); + + return bytes.toByteArray(); + } catch (IOException e) { + return null; + } + } + +} diff --git a/src/main/java/org/qora/network/message/Message.java b/src/main/java/org/qora/network/message/Message.java index a7cd2068..e6de48d0 100644 --- a/src/main/java/org/qora/network/message/Message.java +++ b/src/main/java/org/qora/network/message/Message.java @@ -68,7 +68,9 @@ public abstract class Message { HEIGHT_V2(19), GET_TRANSACTION(20), GET_UNCONFIRMED_TRANSACTIONS(21), - TRANSACTION_SIGNATURES(22); + TRANSACTION_SIGNATURES(22), + GET_ARBITRARY_DATA(23), + ARBITRARY_DATA(24); public final int value; public final Method fromByteBuffer; diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java index 65687c2b..ae497472 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBArbitraryRepository.java @@ -19,6 +19,8 @@ import org.qora.utils.Base58; public class HSQLDBArbitraryRepository implements ArbitraryRepository { + private static final int MAX_RAW_DATA_SIZE = 255; // size of VARBINARY + protected HSQLDBRepository repository; public HSQLDBArbitraryRepository(HSQLDBRepository repository) { @@ -51,30 +53,42 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { return stringBuilder.toString(); } - private String buildPathname(byte[] signature) throws DataException { + private ArbitraryTransactionData getTransactionData(byte[] signature) throws DataException { TransactionData transactionData = this.repository.getTransactionRepository().fromSignature(signature); if (transactionData == null) return null; - return buildPathname((ArbitraryTransactionData) transactionData); + return (ArbitraryTransactionData) transactionData; } @Override public boolean isDataLocal(byte[] signature) throws DataException { - String dataPathname = buildPathname(signature); - if (dataPathname == null) + ArbitraryTransactionData transactionData = getTransactionData(signature); + if (transactionData == null) return false; + // Raw data is always available + if (transactionData.getDataType() == DataType.RAW_DATA) + return true; + + String dataPathname = buildPathname(transactionData); + Path dataPath = Paths.get(dataPathname); return Files.exists(dataPath); } @Override public byte[] fetchData(byte[] signature) throws DataException { - String dataPathname = buildPathname(signature); - if (dataPathname == null) + ArbitraryTransactionData transactionData = getTransactionData(signature); + if (transactionData == null) return null; + // Raw data is always available + if (transactionData.getDataType() == DataType.RAW_DATA) + return transactionData.getData(); + + String dataPathname = buildPathname(transactionData); + Path dataPath = Paths.get(dataPathname); try { return Files.readAllBytes(dataPath); @@ -85,37 +99,47 @@ public class HSQLDBArbitraryRepository implements ArbitraryRepository { @Override public void save(ArbitraryTransactionData arbitraryTransactionData) throws DataException { - // Refuse to store raw data in the repository - it needs to be saved elsewhere! - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) { - byte[] rawData = arbitraryTransactionData.getData(); + // Already hashed? Nothing to do + if (arbitraryTransactionData.getDataType() == DataType.DATA_HASH) + return; - // Calculate hash of data and update our transaction to use that - byte[] dataHash = Crypto.digest(rawData); - arbitraryTransactionData.setData(dataHash); - arbitraryTransactionData.setDataType(DataType.DATA_HASH); + // Trivial-sized payloads can remain in raw form + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA && arbitraryTransactionData.getData().length <= MAX_RAW_DATA_SIZE) + return; - String dataPathname = buildPathname(arbitraryTransactionData); + // Store non-trivial payloads in filesystem and convert transaction's data to hash form + byte[] rawData = arbitraryTransactionData.getData(); - Path dataPath = Paths.get(dataPathname); + // Calculate hash of data and update our transaction to use that + byte[] dataHash = Crypto.digest(rawData); + arbitraryTransactionData.setData(dataHash); + arbitraryTransactionData.setDataType(DataType.DATA_HASH); - // Make sure directory structure exists - try { - Files.createDirectories(dataPath.getParent()); - } catch (IOException e) { - throw new DataException("Unable to create arbitrary transaction directory", e); - } + String dataPathname = buildPathname(arbitraryTransactionData); - // Output actual transaction data - try (OutputStream dataOut = Files.newOutputStream(dataPath)) { - dataOut.write(rawData); - } catch (IOException e) { - throw new DataException("Unable to store arbitrary transaction data", e); - } + Path dataPath = Paths.get(dataPathname); + + // Make sure directory structure exists + try { + Files.createDirectories(dataPath.getParent()); + } catch (IOException e) { + throw new DataException("Unable to create arbitrary transaction directory", e); + } + + // Output actual transaction data + try (OutputStream dataOut = Files.newOutputStream(dataPath)) { + dataOut.write(rawData); + } catch (IOException e) { + throw new DataException("Unable to store arbitrary transaction data", e); } } @Override public void delete(ArbitraryTransactionData arbitraryTransactionData) throws DataException { + // No need to do anything if we still only have raw data, and hence nothing saved in filesystem + if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) + return; + String dataPathname = buildPathname(arbitraryTransactionData); Path dataPath = Paths.get(dataPathname); try { diff --git a/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java b/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java index 6f1edd79..ae80ed03 100644 --- a/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java +++ b/src/main/java/org/qora/repository/hsqldb/HSQLDBDatabaseUpdates.java @@ -758,6 +758,14 @@ public class HSQLDBDatabaseUpdates { stmt.execute("CREATE INDEX TransactionApprovalHeightIndex on Transactions (approval_height)"); break; + case 52: + // Arbitrary transactions changes to allow storage of very small payloads locally + stmt.execute("CREATE TYPE ArbitraryData AS VARBINARY(255)"); + stmt.execute("ALTER TABLE ArbitraryTransactions ADD COLUMN is_data_raw BOOLEAN NOT NULL"); + stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash ArbitraryData"); + stmt.execute("ALTER TABLE ArbitraryTransactions ALTER COLUMN data_hash RENAME TO data"); + break; + default: // nothing to do return false; diff --git a/src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java b/src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java index ba0da2be..2d4b44c8 100644 --- a/src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java +++ b/src/main/java/org/qora/repository/hsqldb/transaction/HSQLDBArbitraryTransactionRepository.java @@ -20,7 +20,7 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos } TransactionData fromBase(BaseTransactionData baseTransactionData) throws DataException { - final String sql = "SELECT version, service, data_hash from ArbitraryTransactions WHERE signature = ?"; + final String sql = "SELECT version, service, is_data_raw, data from ArbitraryTransactions WHERE signature = ?"; try (ResultSet resultSet = this.repository.checkedExecute(sql, baseTransactionData.getSignature())) { if (resultSet == null) @@ -28,11 +28,13 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos int version = resultSet.getInt(1); int service = resultSet.getInt(2); - byte[] dataHash = resultSet.getBytes(3); + boolean isDataRaw = resultSet.getBoolean(3); // NOT NULL, so no null to false + DataType dataType = isDataRaw ? DataType.RAW_DATA : DataType.DATA_HASH; + byte[] data = resultSet.getBytes(4); List payments = this.getPaymentsFromSignature(baseTransactionData.getSignature()); - return new ArbitraryTransactionData(baseTransactionData, version, service, dataHash, DataType.DATA_HASH, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); } catch (SQLException e) { throw new DataException("Unable to fetch arbitrary transaction from repository", e); } @@ -42,15 +44,15 @@ public class HSQLDBArbitraryTransactionRepository extends HSQLDBTransactionRepos public void save(TransactionData transactionData) throws DataException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - // Refuse to store raw data in the repository - it needs to be saved elsewhere! - if (arbitraryTransactionData.getDataType() == DataType.RAW_DATA) + // For V4+, we might not store raw data in the repository but elsewhere + if (arbitraryTransactionData.getVersion() >= 4) this.repository.getArbitraryRepository().save(arbitraryTransactionData); HSQLDBSaver saveHelper = new HSQLDBSaver("ArbitraryTransactions"); saveHelper.bind("signature", arbitraryTransactionData.getSignature()).bind("sender", arbitraryTransactionData.getSenderPublicKey()) .bind("version", arbitraryTransactionData.getVersion()).bind("service", arbitraryTransactionData.getService()) - .bind("data_hash", arbitraryTransactionData.getData()); + .bind("is_data_raw", arbitraryTransactionData.getDataType() == DataType.RAW_DATA).bind("data", arbitraryTransactionData.getData()); try { saveHelper.execute(this.repository); diff --git a/src/main/java/org/qora/transaction/ArbitraryTransaction.java b/src/main/java/org/qora/transaction/ArbitraryTransaction.java index 1a60d2d5..573d9b38 100644 --- a/src/main/java/org/qora/transaction/ArbitraryTransaction.java +++ b/src/main/java/org/qora/transaction/ArbitraryTransaction.java @@ -148,6 +148,7 @@ public class ArbitraryTransaction extends Transaction { return this.repository.getArbitraryRepository().isDataLocal(this.transactionData.getSignature()); } + /** Returns arbitrary data payload, fetching from network if needed. Can block for a while! */ public byte[] fetchData() throws DataException { // If local, read from file if (isDataLocal()) diff --git a/src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java b/src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java index d5d6bb97..513a4fee 100644 --- a/src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java +++ b/src/main/java/org/qora/transform/transaction/ArbitraryTransactionTransformer.java @@ -27,7 +27,9 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { // Property lengths private static final int SERVICE_LENGTH = INT_LENGTH; + private static final int DATA_TYPE_LENGTH = BYTE_LENGTH; private static final int DATA_SIZE_LENGTH = INT_LENGTH; + private static final int NUMBER_PAYMENTS_LENGTH = INT_LENGTH; private static final int EXTRAS_LENGTH = SERVICE_LENGTH + DATA_SIZE_LENGTH; @@ -47,6 +49,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { layout.add("* payment amount", TransformationType.AMOUNT); layout.add("service ID", TransformationType.INT); + layout.add("is data raw?", TransformationType.BOOLEAN); layout.add("data length", TransformationType.INT); layout.add("data", TransformationType.DATA); layout.add("fee", TransformationType.AMOUNT); @@ -78,6 +81,15 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { int service = byteBuffer.getInt(); + // With V4+ we might be receiving hash of data instead of actual raw data + DataType dataType = DataType.RAW_DATA; + if (version >= 4) { + boolean isRaw = byteBuffer.get() != 0; + + if (!isRaw) + dataType = DataType.DATA_HASH; + } + int dataSize = byteBuffer.getInt(); // Don't allow invalid dataSize here to avoid run-time issues if (dataSize > ArbitraryTransaction.MAX_DATA_SIZE) @@ -93,17 +105,21 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { BaseTransactionData baseTransactionData = new BaseTransactionData(timestamp, txGroupId, reference, senderPublicKey, fee, signature); - return new ArbitraryTransactionData(baseTransactionData, version, service, data, DataType.RAW_DATA, payments); + return new ArbitraryTransactionData(baseTransactionData, version, service, data, dataType, payments); } public static int getDataLength(TransactionData transactionData) throws TransformationException { ArbitraryTransactionData arbitraryTransactionData = (ArbitraryTransactionData) transactionData; - int length = getBaseLength(transactionData) + EXTRAS_LENGTH; + int length = getBaseLength(transactionData) + EXTRAS_LENGTH + arbitraryTransactionData.getData().length; + + // V4+ transactions have data type + if (arbitraryTransactionData.getVersion() >= 4) + length += DATA_TYPE_LENGTH; // V3+ transactions have optional payments if (arbitraryTransactionData.getVersion() >= 3) - length += arbitraryTransactionData.getData().length + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); + length += NUMBER_PAYMENTS_LENGTH + arbitraryTransactionData.getPayments().size() * PaymentTransformer.getDataLength(); return length; } @@ -126,6 +142,10 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + // V4+ also has data type + if (arbitraryTransactionData.getVersion() >= 4) + bytes.write((byte) (arbitraryTransactionData.getDataType() == DataType.RAW_DATA ? 1 : 0)); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); bytes.write(arbitraryTransactionData.getData()); @@ -188,6 +208,7 @@ public class ArbitraryTransactionTransformer extends TransactionTransformer { bytes.write(Ints.toByteArray(arbitraryTransactionData.getService())); + bytes.write(Ints.toByteArray(arbitraryTransactionData.getData().length)); switch (arbitraryTransactionData.getDataType()) { case DATA_HASH: bytes.write(arbitraryTransactionData.getData()); diff --git a/src/main/java/org/qora/utils/Serialization.java b/src/main/java/org/qora/utils/Serialization.java index 1598e10d..2587e84d 100644 --- a/src/main/java/org/qora/utils/Serialization.java +++ b/src/main/java/org/qora/utils/Serialization.java @@ -94,7 +94,7 @@ public class Serialization { public static void serializeSizedString(ByteArrayOutputStream bytes, String string) throws UnsupportedEncodingException, IOException { byte[] stringBytes = string.getBytes("UTF-8"); bytes.write(Ints.toByteArray(stringBytes.length)); - bytes.write(string.getBytes("UTF-8")); + bytes.write(stringBytes); } public static String deserializeSizedString(ByteBuffer byteBuffer, int maxSize) throws TransformationException {