From e85026f8666fd3afa621e571c8a0c3cf1bbb9a7d Mon Sep 17 00:00:00 2001 From: catbref Date: Sun, 17 Apr 2022 19:22:29 +0100 Subject: [PATCH] Initial work on reducing network load for transferring blocks. Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states. Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages. Note that blocks fetched from archive are in old V1 format. Changed Triple, List> to BlockTransformation to support both V1 and V2 forms. Set min peer version to 3.3.203 in BlockV2Message class. --- src/main/java/org/qortal/block/Block.java | 67 +++++++- .../org/qortal/controller/Controller.java | 12 ++ .../org/qortal/controller/Synchronizer.java | 26 +-- .../qortal/network/message/BlockMessage.java | 7 +- .../network/message/BlockV2Message.java | 87 ++++++++++ .../qortal/network/message/MessageType.java | 1 + .../qortal/repository/BlockArchiveReader.java | 19 ++- .../hsqldb/HSQLDBBlockArchiveRepository.java | 29 ++-- .../transform/block/BlockTransformation.java | 44 +++++ .../transform/block/BlockTransformer.java | 150 ++++++++++++------ .../org/qortal/utils/BlockArchiveUtils.java | 21 ++- .../org/qortal/test/BlockArchiveTests.java | 19 +-- src/test/java/org/qortal/test/BlockTests.java | 5 +- 13 files changed, 371 insertions(+), 116 deletions(-) create mode 100644 src/main/java/org/qortal/network/message/BlockV2Message.java create mode 100644 src/main/java/org/qortal/transform/block/BlockTransformation.java diff --git a/src/main/java/org/qortal/block/Block.java b/src/main/java/org/qortal/block/Block.java index ea5a6b49..5fe005d6 100644 --- a/src/main/java/org/qortal/block/Block.java +++ b/src/main/java/org/qortal/block/Block.java @@ -3,9 +3,12 @@ package org.qortal.block; import static java.util.Arrays.stream; import static java.util.stream.Collectors.toMap; +import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.math.BigDecimal; import java.math.BigInteger; import java.math.RoundingMode; +import java.nio.charset.StandardCharsets; import java.text.DecimalFormat; import java.text.NumberFormat; import java.util.*; @@ -118,6 +121,8 @@ public class Block { /** Remote/imported/loaded AT states */ protected List atStates; + /** Remote hash of AT states - in lieu of full AT state data in {@code atStates} */ + protected byte[] atStatesHash; /** Locally-generated AT states */ protected List ourAtStates; /** Locally-generated AT fees */ @@ -255,7 +260,7 @@ public class Block { * Constructs new Block using passed transaction and AT states. *

* This constructor typically used when receiving a serialized block over the network. - * + * * @param repository * @param blockData * @param transactions @@ -281,6 +286,35 @@ public class Block { this.blockData.setTotalFees(totalFees); } + /** + * Constructs new Block using passed transaction and minimal AT state info. + *

+ * This constructor typically used when receiving a serialized block over the network. + * + * @param repository + * @param blockData + * @param transactions + * @param atStatesHash + */ + public Block(Repository repository, BlockData blockData, List transactions, byte[] atStatesHash) { + this(repository, blockData); + + this.transactions = new ArrayList<>(); + + long totalFees = 0; + + // We have to sum fees too + for (TransactionData transactionData : transactions) { + this.transactions.add(Transaction.fromData(repository, transactionData)); + totalFees += transactionData.getFee(); + } + + this.atStatesHash = atStatesHash; + totalFees += this.blockData.getATFees(); + + this.blockData.setTotalFees(totalFees); + } + /** * Constructs new Block with empty transaction list, using passed minter account. * @@ -1194,7 +1228,7 @@ public class Block { */ private ValidationResult areAtsValid() throws DataException { // Locally generated AT states should be valid so no need to re-execute them - if (this.ourAtStates == this.getATStates()) // Note object reference compare + if (this.ourAtStates != null && this.ourAtStates == this.atStates) // Note object reference compare return ValidationResult.OK; // Generate local AT states for comparison @@ -1208,8 +1242,33 @@ public class Block { if (this.ourAtFees != this.blockData.getATFees()) return ValidationResult.AT_STATES_MISMATCH; - // Note: this.atStates fully loaded thanks to this.getATStates() call above - for (int s = 0; s < this.atStates.size(); ++s) { + // If we have a single AT states hash then compare that in preference + if (this.atStatesHash != null) { + int atBytesLength = blockData.getATCount() * BlockTransformer.AT_ENTRY_LENGTH; + ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength); + + try { + for (ATStateData atStateData : this.ourAtStates) { + atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8)); + atHashBytes.write(atStateData.getStateHash()); + atHashBytes.write(Longs.toByteArray(atStateData.getFees())); + } + } catch (IOException e) { + throw new DataException("Couldn't validate AT states hash due to serialization issue?", e); + } + + byte[] ourAtStatesHash = Crypto.digest(atHashBytes.toByteArray()); + if (!Arrays.equals(ourAtStatesHash, this.atStatesHash)) + return ValidationResult.AT_STATES_MISMATCH; + + // Use our AT state data from now on + this.atStates = this.ourAtStates; + return ValidationResult.OK; + } + + // Note: this.atStates fully loaded thanks to this.getATStates() call: + this.getATStates(); + for (int s = 0; s < this.ourAtStates.size(); ++s) { ATStateData ourAtState = this.ourAtStates.get(s); ATStateData theirAtState = this.atStates.get(s); diff --git a/src/main/java/org/qortal/controller/Controller.java b/src/main/java/org/qortal/controller/Controller.java index a5ada0c2..0a011db5 100644 --- a/src/main/java/org/qortal/controller/Controller.java +++ b/src/main/java/org/qortal/controller/Controller.java @@ -1362,6 +1362,18 @@ public class Controller extends Thread { Block block = new Block(repository, blockData); + // V2 support + if (peer.getPeersVersion() >= BlockV2Message.MIN_PEER_VERSION) { + Message blockMessage = new BlockV2Message(block); + blockMessage.setId(message.getId()); + if (!peer.sendMessage(blockMessage)) { + peer.disconnect("failed to send block"); + // Don't fall-through to caching because failure to send might be from failure to build message + return; + } + return; + } + CachedBlockMessage blockMessage = new CachedBlockMessage(block); blockMessage.setId(message.getId()); diff --git a/src/main/java/org/qortal/controller/Synchronizer.java b/src/main/java/org/qortal/controller/Synchronizer.java index 8f3a34bb..4c1985a1 100644 --- a/src/main/java/org/qortal/controller/Synchronizer.java +++ b/src/main/java/org/qortal/controller/Synchronizer.java @@ -26,14 +26,7 @@ import org.qortal.event.Event; import org.qortal.event.EventBus; import org.qortal.network.Network; import org.qortal.network.Peer; -import org.qortal.network.message.BlockMessage; -import org.qortal.network.message.BlockSummariesMessage; -import org.qortal.network.message.GetBlockMessage; -import org.qortal.network.message.GetBlockSummariesMessage; -import org.qortal.network.message.GetSignaturesV2Message; -import org.qortal.network.message.Message; -import org.qortal.network.message.SignaturesMessage; -import org.qortal.network.message.MessageType; +import org.qortal.network.message.*; import org.qortal.repository.DataException; import org.qortal.repository.Repository; import org.qortal.repository.RepositoryManager; @@ -1579,12 +1572,23 @@ public class Synchronizer extends Thread { Message getBlockMessage = new GetBlockMessage(signature); Message message = peer.getResponse(getBlockMessage); - if (message == null || message.getType() != MessageType.BLOCK) + if (message == null) return null; - BlockMessage blockMessage = (BlockMessage) message; + switch (message.getType()) { + case BLOCK: { + BlockMessage blockMessage = (BlockMessage) message; + return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); + } - return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); + case BLOCK_V2: { + BlockV2Message blockMessage = (BlockV2Message) message; + return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStatesHash()); + } + + default: + return null; + } } public void populateBlockSummariesMinterLevels(Repository repository, List blockSummaries) throws DataException { diff --git a/src/main/java/org/qortal/network/message/BlockMessage.java b/src/main/java/org/qortal/network/message/BlockMessage.java index 2dd4db87..0a8a23de 100644 --- a/src/main/java/org/qortal/network/message/BlockMessage.java +++ b/src/main/java/org/qortal/network/message/BlockMessage.java @@ -9,6 +9,7 @@ import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformation; import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Triple; @@ -46,12 +47,12 @@ public class BlockMessage extends Message { try { int height = byteBuffer.getInt(); - Triple, List> blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); + BlockTransformation blockTransformation = BlockTransformer.fromByteBuffer(byteBuffer); - BlockData blockData = blockInfo.getA(); + BlockData blockData = blockTransformation.getBlockData(); blockData.setHeight(height); - return new BlockMessage(id, blockData, blockInfo.getB(), blockInfo.getC()); + return new BlockMessage(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStates()); } catch (TransformationException e) { LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage())); throw new MessageException(e.getMessage(), e); diff --git a/src/main/java/org/qortal/network/message/BlockV2Message.java b/src/main/java/org/qortal/network/message/BlockV2Message.java new file mode 100644 index 00000000..815892e2 --- /dev/null +++ b/src/main/java/org/qortal/network/message/BlockV2Message.java @@ -0,0 +1,87 @@ +package org.qortal.network.message; + +import com.google.common.primitives.Ints; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.qortal.block.Block; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; +import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformation; +import org.qortal.transform.block.BlockTransformer; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.List; + +public class BlockV2Message extends Message { + + private static final Logger LOGGER = LogManager.getLogger(BlockV2Message.class); + public static final long MIN_PEER_VERSION = 0x3000300cbL; // 3.3.203 + + private BlockData blockData; + private List transactions; + private byte[] atStatesHash; + + public BlockV2Message(Block block) throws TransformationException { + super(MessageType.BLOCK_V2); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + + try { + bytes.write(Ints.toByteArray(block.getBlockData().getHeight())); + + bytes.write(BlockTransformer.toBytesV2(block)); + } catch (IOException e) { + throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream"); + } + + this.dataBytes = bytes.toByteArray(); + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + public BlockV2Message(byte[] cachedBytes) { + super(MessageType.BLOCK_V2); + + this.dataBytes = cachedBytes; + this.checksumBytes = Message.generateChecksum(this.dataBytes); + } + + private BlockV2Message(int id, BlockData blockData, List transactions, byte[] atStatesHash) { + super(id, MessageType.BLOCK_V2); + + this.blockData = blockData; + this.transactions = transactions; + this.atStatesHash = atStatesHash; + } + + public BlockData getBlockData() { + return this.blockData; + } + + public List getTransactions() { + return this.transactions; + } + + public byte[] getAtStatesHash() { + return this.atStatesHash; + } + + public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException { + try { + int height = byteBuffer.getInt(); + + BlockTransformation blockTransformation = BlockTransformer.fromByteBufferV2(byteBuffer); + + BlockData blockData = blockTransformation.getBlockData(); + blockData.setHeight(height); + + return new BlockV2Message(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStatesHash()); + } catch (TransformationException e) { + LOGGER.info(String.format("Received garbled BLOCK_V2 message: %s", e.getMessage())); + throw new MessageException(e.getMessage(), e); + } + } + +} diff --git a/src/main/java/org/qortal/network/message/MessageType.java b/src/main/java/org/qortal/network/message/MessageType.java index a2637dfd..c2ae7676 100644 --- a/src/main/java/org/qortal/network/message/MessageType.java +++ b/src/main/java/org/qortal/network/message/MessageType.java @@ -34,6 +34,7 @@ public enum MessageType { BLOCK(50, BlockMessage::fromByteBuffer), GET_BLOCK(51, GetBlockMessage::fromByteBuffer), + BLOCK_V2(52, BlockV2Message::fromByteBuffer), SIGNATURES(60, SignaturesMessage::fromByteBuffer), GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer), diff --git a/src/main/java/org/qortal/repository/BlockArchiveReader.java b/src/main/java/org/qortal/repository/BlockArchiveReader.java index 83508152..311d21c7 100644 --- a/src/main/java/org/qortal/repository/BlockArchiveReader.java +++ b/src/main/java/org/qortal/repository/BlockArchiveReader.java @@ -9,6 +9,7 @@ import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; import org.qortal.settings.Settings; import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformation; import org.qortal.transform.block.BlockTransformer; import org.qortal.utils.Triple; @@ -66,7 +67,7 @@ public class BlockArchiveReader { this.fileListCache = Map.copyOf(map); } - public Triple, List> fetchBlockAtHeight(int height) { + public BlockTransformation fetchBlockAtHeight(int height) { if (this.fileListCache == null) { this.fetchFileList(); } @@ -77,13 +78,13 @@ public class BlockArchiveReader { } ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes); - Triple, List> blockInfo = null; + BlockTransformation blockInfo = null; try { blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); - if (blockInfo != null && blockInfo.getA() != null) { + if (blockInfo != null && blockInfo.getBlockData() != null) { // Block height is stored outside of the main serialized bytes, so it // won't be set automatically. - blockInfo.getA().setHeight(height); + blockInfo.getBlockData().setHeight(height); } } catch (TransformationException e) { return null; @@ -91,8 +92,7 @@ public class BlockArchiveReader { return blockInfo; } - public Triple, List> fetchBlockWithSignature( - byte[] signature, Repository repository) { + public BlockTransformation fetchBlockWithSignature(byte[] signature, Repository repository) { if (this.fileListCache == null) { this.fetchFileList(); @@ -105,13 +105,12 @@ public class BlockArchiveReader { return null; } - public List, List>> fetchBlocksFromRange( - int startHeight, int endHeight) { + public List fetchBlocksFromRange(int startHeight, int endHeight) { - List, List>> blockInfoList = new ArrayList<>(); + List blockInfoList = new ArrayList<>(); for (int height = startHeight; height <= endHeight; height++) { - Triple, List> blockInfo = this.fetchBlockAtHeight(height); + BlockTransformation blockInfo = this.fetchBlockAtHeight(height); if (blockInfo == null) { return blockInfoList; } diff --git a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java index d8738f0d..cc7e1611 100644 --- a/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java +++ b/src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java @@ -1,16 +1,13 @@ package org.qortal.repository.hsqldb; -import org.qortal.api.ApiError; -import org.qortal.api.ApiExceptionFactory; import org.qortal.api.model.BlockSignerSummary; -import org.qortal.block.Block; import org.qortal.data.block.BlockArchiveData; import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockSummaryData; import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.BlockArchiveRepository; import org.qortal.repository.DataException; -import org.qortal.utils.Triple; +import org.qortal.transform.block.BlockTransformation; import java.sql.ResultSet; import java.sql.SQLException; @@ -29,11 +26,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { @Override public BlockData fromSignature(byte[] signature) throws DataException { - Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); - if (blockInfo != null) { - return (BlockData) blockInfo.getA(); - } - return null; + BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); + if (blockInfo == null) + return null; + + return blockInfo.getBlockData(); } @Override @@ -47,11 +44,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { @Override public BlockData fromHeight(int height) throws DataException { - Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); - if (blockInfo != null) { - return (BlockData) blockInfo.getA(); - } - return null; + BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); + if (blockInfo == null) + return null; + + return blockInfo.getBlockData(); } @Override @@ -79,9 +76,9 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository { int height = referenceBlock.getHeight(); if (height > 0) { // Request the block at height + 1 - Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); + BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); if (blockInfo != null) { - return (BlockData) blockInfo.getA(); + return blockInfo.getBlockData(); } } } diff --git a/src/main/java/org/qortal/transform/block/BlockTransformation.java b/src/main/java/org/qortal/transform/block/BlockTransformation.java new file mode 100644 index 00000000..6aee8cf9 --- /dev/null +++ b/src/main/java/org/qortal/transform/block/BlockTransformation.java @@ -0,0 +1,44 @@ +package org.qortal.transform.block; + +import org.qortal.data.at.ATStateData; +import org.qortal.data.block.BlockData; +import org.qortal.data.transaction.TransactionData; + +import java.util.List; + +public class BlockTransformation { + private final BlockData blockData; + private final List transactions; + private final List atStates; + private final byte[] atStatesHash; + + /*package*/ BlockTransformation(BlockData blockData, List transactions, List atStates) { + this.blockData = blockData; + this.transactions = transactions; + this.atStates = atStates; + this.atStatesHash = null; + } + + /*package*/ BlockTransformation(BlockData blockData, List transactions, byte[] atStatesHash) { + this.blockData = blockData; + this.transactions = transactions; + this.atStates = null; + this.atStatesHash = atStatesHash; + } + + public BlockData getBlockData() { + return blockData; + } + + public List getTransactions() { + return transactions; + } + + public List getAtStates() { + return atStates; + } + + public byte[] getAtStatesHash() { + return atStatesHash; + } +} diff --git a/src/main/java/org/qortal/transform/block/BlockTransformer.java b/src/main/java/org/qortal/transform/block/BlockTransformer.java index cce3e7d7..b61d6900 100644 --- a/src/main/java/org/qortal/transform/block/BlockTransformer.java +++ b/src/main/java/org/qortal/transform/block/BlockTransformer.java @@ -3,12 +3,14 @@ package org.qortal.transform.block; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.qortal.block.Block; import org.qortal.block.BlockChain; +import org.qortal.crypto.Crypto; import org.qortal.data.at.ATStateData; import org.qortal.data.block.BlockData; import org.qortal.data.transaction.TransactionData; @@ -20,7 +22,6 @@ import org.qortal.transform.Transformer; import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; import org.qortal.utils.Serialization; -import org.qortal.utils.Triple; import com.google.common.primitives.Ints; import com.google.common.primitives.Longs; @@ -45,14 +46,13 @@ public class BlockTransformer extends Transformer { protected static final int AT_BYTES_LENGTH = INT_LENGTH; protected static final int AT_FEES_LENGTH = AMOUNT_LENGTH; - protected static final int AT_LENGTH = AT_FEES_LENGTH + AT_BYTES_LENGTH; protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = TIMESTAMP_LENGTH; protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH; - protected static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH; + public static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH; /** * Extract block data and transaction data from serialized bytes. @@ -61,7 +61,7 @@ public class BlockTransformer extends Transformer { * @return BlockData and a List of transactions. * @throws TransformationException */ - public static Triple, List> fromBytes(byte[] bytes) throws TransformationException { + public static BlockTransformation fromBytes(byte[] bytes) throws TransformationException { if (bytes == null) return null; @@ -76,28 +76,40 @@ public class BlockTransformer extends Transformer { /** * Extract block data and transaction data from serialized bytes containing a single block. * - * @param bytes + * @param byteBuffer source of serialized block bytes * @return BlockData and a List of transactions. * @throws TransformationException */ - public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + public static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { + return BlockTransformer.fromByteBuffer(byteBuffer, false); + } + + /** + * Extract block data and transaction data from serialized bytes containing a single block. + * + * @param byteBuffer source of serialized block bytes + * @return BlockData and a List of transactions. + * @throws TransformationException + */ + public static BlockTransformation fromByteBufferV2(ByteBuffer byteBuffer) throws TransformationException { return BlockTransformer.fromByteBuffer(byteBuffer, true); } /** - * Extract block data and transaction data from serialized bytes containing one or more blocks. - * - * @param bytes + * Extract block data and transaction data from serialized bytes containing a single block, in one of two forms. + * + * @param byteBuffer source of serialized block bytes + * @param isV2 set to true if AT state info is represented by a single hash, false if serialized as per-AT address+state hash+fees * @return the next block's BlockData and a List of transactions. * @throws TransformationException */ - public static Triple, List> fromByteBuffer(ByteBuffer byteBuffer, boolean finalBlockInBuffer) throws TransformationException { + private static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer, boolean isV2) throws TransformationException { int version = byteBuffer.getInt(); - if (finalBlockInBuffer && byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) + if (byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) throw new TransformationException("Byte data too short for Block"); - if (finalBlockInBuffer && byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) + if (byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) throw new TransformationException("Byte data too long for Block"); long timestamp = byteBuffer.getLong(); @@ -117,42 +129,52 @@ public class BlockTransformer extends Transformer { int atCount = 0; long atFees = 0; - List atStates = new ArrayList<>(); + byte[] atStatesHash = null; + List atStates = null; - int atBytesLength = byteBuffer.getInt(); + if (isV2) { + // Simply: AT count, AT total fees, hash(all AT states) + atCount = byteBuffer.getInt(); + atFees = byteBuffer.getLong(); + atStatesHash = new byte[Transformer.SHA256_LENGTH]; + byteBuffer.get(atStatesHash); + } else { + // V1: AT info byte length, then per-AT entries of AT address + state hash + fees + int atBytesLength = byteBuffer.getInt(); + if (atBytesLength > BlockChain.getInstance().getMaxBlockSize()) + throw new TransformationException("Byte data too long for Block's AT info"); - if (atBytesLength > BlockChain.getInstance().getMaxBlockSize()) - throw new TransformationException("Byte data too long for Block's AT info"); + // Read AT-address, SHA256 hash and fees + if (atBytesLength % AT_ENTRY_LENGTH != 0) + throw new TransformationException("AT byte data not a multiple of AT entry length"); - ByteBuffer atByteBuffer = byteBuffer.slice(); - atByteBuffer.limit(atBytesLength); + ByteBuffer atByteBuffer = byteBuffer.slice(); + atByteBuffer.limit(atBytesLength); - // Read AT-address, SHA256 hash and fees - if (atBytesLength % AT_ENTRY_LENGTH != 0) - throw new TransformationException("AT byte data not a multiple of AT entry length"); + atStates = new ArrayList<>(); + while (atByteBuffer.hasRemaining()) { + byte[] atAddressBytes = new byte[ADDRESS_LENGTH]; + atByteBuffer.get(atAddressBytes); + String atAddress = Base58.encode(atAddressBytes); - while (atByteBuffer.hasRemaining()) { - byte[] atAddressBytes = new byte[ADDRESS_LENGTH]; - atByteBuffer.get(atAddressBytes); - String atAddress = Base58.encode(atAddressBytes); + byte[] stateHash = new byte[SHA256_LENGTH]; + atByteBuffer.get(stateHash); - byte[] stateHash = new byte[SHA256_LENGTH]; - atByteBuffer.get(stateHash); + long fees = atByteBuffer.getLong(); - long fees = atByteBuffer.getLong(); + // Add this AT's fees to our total + atFees += fees; - // Add this AT's fees to our total - atFees += fees; + atStates.add(new ATStateData(atAddress, stateHash, fees)); + } - atStates.add(new ATStateData(atAddress, stateHash, fees)); + // Bump byteBuffer over AT states just read in slice + byteBuffer.position(byteBuffer.position() + atBytesLength); + + // AT count to reflect the number of states we have + atCount = atStates.size(); } - // Bump byteBuffer over AT states just read in slice - byteBuffer.position(byteBuffer.position() + atBytesLength); - - // AT count to reflect the number of states we have - atCount = atStates.size(); - // Add AT fees to totalFees totalFees += atFees; @@ -221,16 +243,15 @@ public class BlockTransformer extends Transformer { byteBuffer.get(onlineAccountsSignatures); } - // We should only complain about excess byte data if we aren't expecting more blocks in this ByteBuffer - if (finalBlockInBuffer && byteBuffer.hasRemaining()) - throw new TransformationException("Excess byte data found after parsing Block"); - // We don't have a height! Integer height = null; BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, minterPublicKey, minterSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); - return new Triple<>(blockData, transactions, atStates); + if (isV2) + return new BlockTransformation(blockData, transactions, atStatesHash); + else + return new BlockTransformation(blockData, transactions, atStates); } public static int getDataLength(Block block) throws TransformationException { @@ -266,6 +287,14 @@ public class BlockTransformer extends Transformer { } public static byte[] toBytes(Block block) throws TransformationException { + return toBytes(block, false); + } + + public static byte[] toBytesV2(Block block) throws TransformationException { + return toBytes(block, true); + } + + private static byte[] toBytes(Block block, boolean isV2) throws TransformationException { BlockData blockData = block.getBlockData(); try { @@ -279,16 +308,37 @@ public class BlockTransformer extends Transformer { bytes.write(blockData.getMinterSignature()); int atBytesLength = blockData.getATCount() * AT_ENTRY_LENGTH; - bytes.write(Ints.toByteArray(atBytesLength)); + if (isV2) { + ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength); + long atFees = 0; - for (ATStateData atStateData : block.getATStates()) { - // Skip initial states generated by DEPLOY_AT transactions in the same block - if (atStateData.isInitial()) - continue; + for (ATStateData atStateData : block.getATStates()) { + // Skip initial states generated by DEPLOY_AT transactions in the same block + if (atStateData.isInitial()) + continue; - bytes.write(Base58.decode(atStateData.getATAddress())); - bytes.write(atStateData.getStateHash()); - bytes.write(Longs.toByteArray(atStateData.getFees())); + atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8)); + atHashBytes.write(atStateData.getStateHash()); + atHashBytes.write(Longs.toByteArray(atStateData.getFees())); + + atFees += atStateData.getFees(); + } + + bytes.write(Ints.toByteArray(blockData.getATCount())); + bytes.write(Longs.toByteArray(atFees)); + bytes.write(Crypto.digest(atHashBytes.toByteArray())); + } else { + bytes.write(Ints.toByteArray(atBytesLength)); + + for (ATStateData atStateData : block.getATStates()) { + // Skip initial states generated by DEPLOY_AT transactions in the same block + if (atStateData.isInitial()) + continue; + + bytes.write(Base58.decode(atStateData.getATAddress())); + bytes.write(atStateData.getStateHash()); + bytes.write(Longs.toByteArray(atStateData.getFees())); + } } // Transactions diff --git a/src/main/java/org/qortal/utils/BlockArchiveUtils.java b/src/main/java/org/qortal/utils/BlockArchiveUtils.java index 0beff026..84de1a31 100644 --- a/src/main/java/org/qortal/utils/BlockArchiveUtils.java +++ b/src/main/java/org/qortal/utils/BlockArchiveUtils.java @@ -6,6 +6,7 @@ import org.qortal.data.transaction.TransactionData; import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.DataException; import org.qortal.repository.Repository; +import org.qortal.transform.block.BlockTransformation; import java.util.List; @@ -33,8 +34,7 @@ public class BlockArchiveUtils { repository.discardChanges(); final int requestedRange = endHeight+1-startHeight; - List, List>> blockInfoList = - BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight); + List blockInfoList = BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight); // Ensure that we have received all of the requested blocks if (blockInfoList == null || blockInfoList.isEmpty()) { @@ -43,27 +43,26 @@ public class BlockArchiveUtils { if (blockInfoList.size() != requestedRange) { throw new IllegalStateException("Non matching block count when importing from archive"); } - Triple, List> firstBlock = blockInfoList.get(0); - if (firstBlock == null || firstBlock.getA().getHeight() != startHeight) { + BlockTransformation firstBlock = blockInfoList.get(0); + if (firstBlock == null || firstBlock.getBlockData().getHeight() != startHeight) { throw new IllegalStateException("Non matching first block when importing from archive"); } if (blockInfoList.size() > 0) { - Triple, List> lastBlock = - blockInfoList.get(blockInfoList.size() - 1); - if (lastBlock == null || lastBlock.getA().getHeight() != endHeight) { + BlockTransformation lastBlock = blockInfoList.get(blockInfoList.size() - 1); + if (lastBlock == null || lastBlock.getBlockData().getHeight() != endHeight) { throw new IllegalStateException("Non matching last block when importing from archive"); } } // Everything seems okay, so go ahead with the import - for (Triple, List> blockInfo : blockInfoList) { + for (BlockTransformation blockInfo : blockInfoList) { try { // Save block - repository.getBlockRepository().save(blockInfo.getA()); + repository.getBlockRepository().save(blockInfo.getBlockData()); // Save AT state data hashes - for (ATStateData atStateData : blockInfo.getC()) { - atStateData.setHeight(blockInfo.getA().getHeight()); + for (ATStateData atStateData : blockInfo.getAtStates()) { + atStateData.setHeight(blockInfo.getBlockData().getHeight()); repository.getATRepository().save(atStateData); } diff --git a/src/test/java/org/qortal/test/BlockArchiveTests.java b/src/test/java/org/qortal/test/BlockArchiveTests.java index e2f2ed1c..32fd0283 100644 --- a/src/test/java/org/qortal/test/BlockArchiveTests.java +++ b/src/test/java/org/qortal/test/BlockArchiveTests.java @@ -20,6 +20,7 @@ import org.qortal.test.common.Common; import org.qortal.transaction.DeployAtTransaction; import org.qortal.transaction.Transaction; import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformation; import org.qortal.utils.BlockArchiveUtils; import org.qortal.utils.NTP; import org.qortal.utils.Triple; @@ -123,8 +124,8 @@ public class BlockArchiveTests extends Common { // Read block 2 from the archive BlockArchiveReader reader = BlockArchiveReader.getInstance(); - Triple, List> block2Info = reader.fetchBlockAtHeight(2); - BlockData block2ArchiveData = block2Info.getA(); + BlockTransformation block2Info = reader.fetchBlockAtHeight(2); + BlockData block2ArchiveData = block2Info.getBlockData(); // Read block 2 from the repository BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2); @@ -137,8 +138,8 @@ public class BlockArchiveTests extends Common { assertEquals(1, block2ArchiveData.getOnlineAccountsCount()); // Read block 900 from the archive - Triple, List> block900Info = reader.fetchBlockAtHeight(900); - BlockData block900ArchiveData = block900Info.getA(); + BlockTransformation block900Info = reader.fetchBlockAtHeight(900); + BlockData block900ArchiveData = block900Info.getBlockData(); // Read block 900 from the repository BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900); @@ -200,10 +201,10 @@ public class BlockArchiveTests extends Common { // Read a block from the archive BlockArchiveReader reader = BlockArchiveReader.getInstance(); - Triple, List> blockInfo = reader.fetchBlockAtHeight(testHeight); - BlockData archivedBlockData = blockInfo.getA(); - ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0); - List archivedTransactions = blockInfo.getB(); + BlockTransformation blockInfo = reader.fetchBlockAtHeight(testHeight); + BlockData archivedBlockData = blockInfo.getBlockData(); + ATStateData archivedAtStateData = blockInfo.getAtStates().isEmpty() ? null : blockInfo.getAtStates().get(0); + List archivedTransactions = blockInfo.getTransactions(); // Read the same block from the repository BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight); @@ -255,7 +256,7 @@ public class BlockArchiveTests extends Common { // Check block 10 (unarchived) BlockArchiveReader reader = BlockArchiveReader.getInstance(); - Triple, List> blockInfo = reader.fetchBlockAtHeight(10); + BlockTransformation blockInfo = reader.fetchBlockAtHeight(10); assertNull(blockInfo); } diff --git a/src/test/java/org/qortal/test/BlockTests.java b/src/test/java/org/qortal/test/BlockTests.java index d6fdac02..53b216ec 100644 --- a/src/test/java/org/qortal/test/BlockTests.java +++ b/src/test/java/org/qortal/test/BlockTests.java @@ -24,6 +24,7 @@ import org.qortal.test.common.TransactionUtils; import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transform.TransformationException; +import org.qortal.transform.block.BlockTransformation; import org.qortal.transform.block.BlockTransformer; import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.utils.Base58; @@ -121,10 +122,10 @@ public class BlockTests extends Common { assertEquals(BlockTransformer.getDataLength(block), bytes.length); - Triple, List> blockInfo = BlockTransformer.fromBytes(bytes); + BlockTransformation blockInfo = BlockTransformer.fromBytes(bytes); // Compare transactions - List deserializedTransactions = blockInfo.getB(); + List deserializedTransactions = blockInfo.getTransactions(); assertEquals("Transaction count differs", blockData.getTransactionCount(), deserializedTransactions.size()); for (int i = 0; i < blockData.getTransactionCount(); ++i) {