From 21d7a4eed16454db1d0f920bae839cdfb27e3d44 Mon Sep 17 00:00:00 2001 From: catbref Date: Fri, 17 Jul 2020 11:46:39 +0100 Subject: [PATCH] Improved AT PUT_TX_AFTER_TIMESTAMP_INTO_A function Previous version fetched all the blocks from previous 'timestamp' to current height, checking each transaction. (very slow) New implementation leverages repository to do the heavy lifting. Could potentially benefit from some DB indexes in the future? Added unit test to cover. --- src/main/java/org/qortal/at/QortalATAPI.java | 70 ++--- .../org/qortal/repository/ATRepository.java | 24 ++ .../repository/hsqldb/HSQLDBATRepository.java | 36 +++ .../test/at/GetNextTransactionTests.java | 268 ++++++++++++++++++ 4 files changed, 346 insertions(+), 52 deletions(-) create mode 100644 src/test/java/org/qortal/test/at/GetNextTransactionTests.java diff --git a/src/main/java/org/qortal/at/QortalATAPI.java b/src/main/java/org/qortal/at/QortalATAPI.java index 0975bacb..582b44e2 100644 --- a/src/main/java/org/qortal/at/QortalATAPI.java +++ b/src/main/java/org/qortal/at/QortalATAPI.java @@ -17,7 +17,6 @@ import org.qortal.account.Account; import org.qortal.account.NullAccount; import org.qortal.account.PublicKeyAccount; import org.qortal.asset.Asset; -import org.qortal.block.Block; import org.qortal.block.BlockChain; import org.qortal.block.BlockChain.CiyamAtSettings; import org.qortal.crypto.Crypto; @@ -30,11 +29,10 @@ import org.qortal.data.transaction.MessageTransactionData; import org.qortal.data.transaction.PaymentTransactionData; import org.qortal.data.transaction.TransactionData; import org.qortal.group.Group; -import org.qortal.repository.BlockRepository; +import org.qortal.repository.ATRepository; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.transaction.AtTransaction; -import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.utils.Base58; import org.qortal.utils.BitTwiddling; @@ -150,59 +148,27 @@ public class QortalATAPI extends API { int height = timestamp.blockHeight; int sequence = timestamp.transactionSequence + 1; - BlockRepository blockRepository = this.getRepository().getBlockRepository(); - + ATRepository.NextTransactionInfo nextTransactionInfo; try { - int currentHeight = blockRepository.getBlockchainHeight(); - List blockTransactions = null; - - while (height <= currentHeight) { - if (blockTransactions == null) { - BlockData blockData = blockRepository.fromHeight(height); - - if (blockData == null) - throw new DataException("Unable to fetch block " + height + " from repository?"); - - Block block = new Block(this.getRepository(), blockData); - - blockTransactions = block.getTransactions(); - } - - // No more transactions in this block? Try next block - if (sequence >= blockTransactions.size()) { - ++height; - sequence = 0; - blockTransactions = null; - continue; - } - - Transaction transaction = blockTransactions.get(sequence); - - // Transaction needs to be sent to specified recipient - List recipientAddresses = transaction.getRecipientAddresses(); - if (recipientAddresses.contains(atAddress)) { - // Found a transaction - - this.setA1(state, new Timestamp(height, timestamp.blockchainId, sequence).longValue()); - - // Copy transaction's partial signature into the other three A fields for future verification that it's the same transaction - byte[] signature = transaction.getTransactionData().getSignature(); - this.setA2(state, BitTwiddling.longFromBEBytes(signature, 8)); - this.setA3(state, BitTwiddling.longFromBEBytes(signature, 16)); - this.setA4(state, BitTwiddling.longFromBEBytes(signature, 24)); - - return; - } - - // Transaction wasn't for us - keep going - ++sequence; - } - - // No more transactions - zero A and exit - this.zeroA(state); + nextTransactionInfo = this.getRepository().getATRepository().findNextTransaction(atAddress, height, sequence); } catch (DataException e) { throw new RuntimeException("AT API unable to fetch next transaction?", e); } + + if (nextTransactionInfo == null) { + // No more transactions for AT at this time - zero A and exit + this.zeroA(state); + return; + } + + // Found a transaction + + this.setA1(state, new Timestamp(nextTransactionInfo.height, timestamp.blockchainId, nextTransactionInfo.sequence).longValue()); + + // Copy transaction's partial signature into the other three A fields for future verification that it's the same transaction + this.setA2(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 8)); + this.setA3(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 16)); + this.setA4(state, BitTwiddling.longFromBEBytes(nextTransactionInfo.signature, 24)); } @Override diff --git a/src/main/java/org/qortal/repository/ATRepository.java b/src/main/java/org/qortal/repository/ATRepository.java index affbaf18..887672d8 100644 --- a/src/main/java/org/qortal/repository/ATRepository.java +++ b/src/main/java/org/qortal/repository/ATRepository.java @@ -88,4 +88,28 @@ public interface ATRepository { /** Delete state data for all ATs at this height */ public void deleteATStates(int height) throws DataException; + // Finding transactions for ATs to process + + static class NextTransactionInfo { + public final int height; + public final int sequence; + public final byte[] signature; + + public NextTransactionInfo(int height, int sequence, byte[] signature) { + this.height = height; + this.sequence = sequence; + this.signature = signature; + } + } + + /** + * Find next transaction for AT to process. + *

+ * @param recipient AT address + * @param height starting height + * @param sequence starting sequence + * @return next transaction info, or null if none found + */ + public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException; + } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java index f6de4fb4..808cc44d 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBATRepository.java @@ -341,4 +341,40 @@ public class HSQLDBATRepository implements ATRepository { } } + // Finding transactions for ATs to process + + public NextTransactionInfo findNextTransaction(String recipient, int height, int sequence) throws DataException { + // We only need to search for a subset of transaction types: MESSAGE, PAYMENT or AT + + String sql = "SELECT height, sequence, Transactions.signature " + + "FROM (" + + "SELECT signature FROM PaymentTransactions WHERE recipient = ? " + + "UNION " + + "SELECT signature FROM MessageTransactions WHERE recipient = ? " + + "UNION " + + "SELECT signature FROM ATTransactions WHERE recipient = ?" + + ") AS Transactions " + + "JOIN BlockTransactions ON BlockTransactions.transaction_signature = Transactions.signature " + + "JOIN Blocks ON Blocks.signature = BlockTransactions.block_signature " + + "WHERE (height > ? OR (height = ? AND sequence > ?)) " + + "ORDER BY height ASC, sequence ASC " + + "LIMIT 1"; + + Object[] bindParams = new Object[] { recipient, recipient, recipient, height, height, sequence }; + + try (ResultSet resultSet = this.repository.checkedExecute(sql, bindParams)) { + if (resultSet == null) + return null; + + int nextHeight = resultSet.getInt(1); + int nextSequence = resultSet.getInt(2); + byte[] nextSignature = resultSet.getBytes(3); + + return new NextTransactionInfo(nextHeight, nextSequence, nextSignature); + } catch (SQLException e) { + throw new DataException("Unable to find next transaction to AT from repository", e); + } + + } + } diff --git a/src/test/java/org/qortal/test/at/GetNextTransactionTests.java b/src/test/java/org/qortal/test/at/GetNextTransactionTests.java new file mode 100644 index 00000000..eafc22fb --- /dev/null +++ b/src/test/java/org/qortal/test/at/GetNextTransactionTests.java @@ -0,0 +1,268 @@ +package org.qortal.test.at; + +import static org.junit.Assert.*; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.List; + +import org.ciyam.at.CompilationException; +import org.ciyam.at.FunctionCode; +import org.ciyam.at.MachineState; +import org.ciyam.at.OpCode; +import org.ciyam.at.Timestamp; +import org.junit.Before; +import org.junit.Test; +import org.qortal.account.PrivateKeyAccount; +import org.qortal.asset.Asset; +import org.qortal.at.QortalAtLoggerFactory; +import org.qortal.block.Block; +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.BaseTransactionData; +import org.qortal.data.transaction.DeployAtTransactionData; +import org.qortal.data.transaction.MessageTransactionData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.group.Group; +import org.qortal.repository.DataException; +import org.qortal.repository.Repository; +import org.qortal.repository.RepositoryManager; +import org.qortal.test.common.BlockUtils; +import org.qortal.test.common.Common; +import org.qortal.test.common.TransactionUtils; +import org.qortal.transaction.DeployAtTransaction; +import org.qortal.transaction.MessageTransaction; +import org.qortal.transaction.Transaction; +import org.qortal.utils.BitTwiddling; + +public class GetNextTransactionTests extends Common { + + @Before + public void before() throws DataException { + Common.useDefaultSettings(); + } + + @Test + public void testGetNextTransaction() throws DataException { + byte[] data = new byte[] { 0x44 }; + + try (final Repository repository = RepositoryManager.getRepository()) { + PrivateKeyAccount deployer = Common.getTestAccount(repository, "alice"); + + byte[] creationBytes = buildGetNextTransactionAT(); + + long fundingAmount = 1_00000000L; + DeployAtTransaction deployAtTransaction = doDeploy(repository, deployer, creationBytes, fundingAmount); + String atAddress = deployAtTransaction.getATAccount().getAddress(); + + byte[] rawNextTimestamp = new byte[32]; + Transaction transaction; + + // Confirm initial value is zero + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + // Send message to someone other than AT + sendMessage(repository, deployer, data, deployer.getAddress()); + BlockUtils.mintBlock(repository); + + // Confirm AT does not find message + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + assertArrayEquals(new byte[32], rawNextTimestamp); + + // Send message to AT + transaction = sendMessage(repository, deployer, data, atAddress); + BlockUtils.mintBlock(repository); + + // Confirm AT finds message + BlockUtils.mintBlock(repository); + assertTimestamp(repository, atAddress, transaction); + + // Mint a few blocks, then send non-AT message, followed by AT message + for (int i = 0; i < 5; ++i) + BlockUtils.mintBlock(repository); + sendMessage(repository, deployer, data, deployer.getAddress()); + transaction = sendMessage(repository, deployer, data, atAddress); + BlockUtils.mintBlock(repository); + + // Confirm AT finds message + BlockUtils.mintBlock(repository); + assertTimestamp(repository, atAddress, transaction); + } + } + + private byte[] buildGetNextTransactionAT() { + // Labels for data segment addresses + int addrCounter = 0; + + // Beginning of data segment for easy extraction + final int addrNextTx = addrCounter; + addrCounter += 4; + + final int addrNextTxIndex = addrCounter++; + + final int addrLastTxTimestamp = addrCounter++; + + // Data segment + ByteBuffer dataByteBuffer = ByteBuffer.allocate(addrCounter * MachineState.VALUE_SIZE); + + // skip addrNextTx + dataByteBuffer.position(dataByteBuffer.position() + 4 * MachineState.VALUE_SIZE); + + // Store pointer to addrNextTx at addrNextTxIndex + dataByteBuffer.putLong(addrNextTx); + + ByteBuffer codeByteBuffer = ByteBuffer.allocate(512); + + // Two-pass version + for (int pass = 0; pass < 2; ++pass) { + codeByteBuffer.clear(); + + try { + /* Initialization */ + + // Use AT creation 'timestamp' as starting point for finding transactions sent to AT + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_CREATION_TIMESTAMP, addrLastTxTimestamp)); + + // Set restart position to after this opcode + codeByteBuffer.put(OpCode.SET_PCS.compile()); + + /* Loop, waiting for message to AT */ + + // Find next transaction to this AT since the last one (if any) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.PUT_TX_AFTER_TIMESTAMP_INTO_A, addrLastTxTimestamp)); + // Copy A to data segment, starting at addrNextTx (as pointed to by addrNextTxIndex) + codeByteBuffer.put(OpCode.EXT_FUN_DAT.compile(FunctionCode.GET_A_IND, addrNextTxIndex)); + // Stop if timestamp part of A is zero + codeByteBuffer.put(OpCode.STZ_DAT.compile(addrNextTx)); + + // Update our 'last found transaction's timestamp' using 'timestamp' from transaction + codeByteBuffer.put(OpCode.EXT_FUN_RET.compile(FunctionCode.GET_TIMESTAMP_FROM_TX_IN_A, addrLastTxTimestamp)); + // Stop and wait for next block + codeByteBuffer.put(OpCode.STP_IMD.compile()); + + } catch (CompilationException e) { + throw new IllegalStateException("Unable to compile AT?", e); + } + } + + codeByteBuffer.flip(); + + byte[] codeBytes = new byte[codeByteBuffer.limit()]; + codeByteBuffer.get(codeBytes); + + final short ciyamAtVersion = 2; + final short numCallStackPages = 0; + final short numUserStackPages = 0; + final long minActivationAmount = 0L; + + return MachineState.toCreationBytes(ciyamAtVersion, codeBytes, dataByteBuffer.array(), numCallStackPages, numUserStackPages, minActivationAmount); + } + + private DeployAtTransaction doDeploy(Repository repository, PrivateKeyAccount deployer, byte[] creationBytes, long fundingAmount) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = deployer.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", deployer.getAddress())); + System.exit(2); + } + + Long fee = null; + String name = "Test AT"; + String description = "Test AT"; + String atType = "Test"; + String tags = "TEST"; + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, deployer.getPublicKey(), fee, null); + TransactionData deployAtTransactionData = new DeployAtTransactionData(baseTransactionData, name, description, atType, tags, creationBytes, fundingAmount, Asset.QORT); + + DeployAtTransaction deployAtTransaction = new DeployAtTransaction(repository, deployAtTransactionData); + + fee = deployAtTransaction.calcRecommendedFee(); + deployAtTransactionData.setFee(fee); + + TransactionUtils.signAndMint(repository, deployAtTransactionData, deployer); + + return deployAtTransaction; + } + + private void extractNextTxTimestamp(Repository repository, String atAddress, byte[] rawNextTimestamp) throws DataException { + // Check AT result + ATStateData atStateData = repository.getATRepository().getLatestATState(atAddress); + byte[] stateData = atStateData.getStateData(); + + QortalAtLoggerFactory loggerFactory = QortalAtLoggerFactory.getInstance(); + byte[] dataBytes = MachineState.extractDataBytes(loggerFactory, stateData); + + System.arraycopy(dataBytes, 0, rawNextTimestamp, 0, rawNextTimestamp.length); + } + + private MessageTransaction sendMessage(Repository repository, PrivateKeyAccount sender, byte[] data, String recipient) throws DataException { + long txTimestamp = System.currentTimeMillis(); + byte[] lastReference = sender.getLastReference(); + + if (lastReference == null) { + System.err.println(String.format("Qortal account %s has no last reference", sender.getAddress())); + System.exit(2); + } + + Long fee = null; + int version = 4; + int nonce = 0; + long amount = 0; + Long assetId = null; // because amount is zero + + BaseTransactionData baseTransactionData = new BaseTransactionData(txTimestamp, Group.NO_GROUP, lastReference, sender.getPublicKey(), fee, null); + TransactionData messageTransactionData = new MessageTransactionData(baseTransactionData, version, nonce, recipient, amount, assetId, data, false, false); + + MessageTransaction messageTransaction = new MessageTransaction(repository, messageTransactionData); + + fee = messageTransaction.calcRecommendedFee(); + messageTransactionData.setFee(fee); + + TransactionUtils.signAndImportValid(repository, messageTransactionData, sender); + + return messageTransaction; + } + + private void assertTimestamp(Repository repository, String atAddress, Transaction transaction) throws DataException { + int height = transaction.getHeight(); + byte[] transactionSignature = transaction.getTransactionData().getSignature(); + + BlockData blockData = repository.getBlockRepository().fromHeight(height); + assertNotNull(blockData); + + Block block = new Block(repository, blockData); + + List blockTransactions = block.getTransactions(); + int sequence; + for (sequence = blockTransactions.size() - 1; sequence >= 0; --sequence) + if (Arrays.equals(blockTransactions.get(sequence).getTransactionData().getSignature(), transactionSignature)) + break; + + assertNotSame(-1, sequence); + + byte[] rawNextTimestamp = new byte[32]; + extractNextTxTimestamp(repository, atAddress, rawNextTimestamp); + + Timestamp expectedTimestamp = new Timestamp(height, sequence); + Timestamp actualTimestamp = new Timestamp(BitTwiddling.longFromBEBytes(rawNextTimestamp, 0)); + + assertEquals(String.format("Expected height %d, seq %d but was height %d, seq %d", + height, sequence, + actualTimestamp.blockHeight, actualTimestamp.transactionSequence + ), + expectedTimestamp.longValue(), + actualTimestamp.longValue()); + + byte[] expectedPartialSignature = new byte[24]; + System.arraycopy(transactionSignature, 8, expectedPartialSignature, 0, expectedPartialSignature.length); + + byte[] actualPartialSignature = new byte[24]; + System.arraycopy(rawNextTimestamp, 8, actualPartialSignature, 0, actualPartialSignature.length); + + assertArrayEquals(expectedPartialSignature, actualPartialSignature); + } + +}