crowetic/qortal
1
0
Fork 0
forked from Qortal/qortal
Code Pull Requests Activity

Initial work on reducing network load for transferring blocks.

Browse Source 
Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
This commit is contained in:
catbref 2022-04-17 19:22:29 +01:00
parent 987446cf7f
commit 6950c6bf69
13 changed files with 371 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

67
src/main/java/org/qortal/block/Block.java
View File

@ -3,9 +3,12 @@ package org.qortal.block;
import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.*;
@ -118,6 +121,8 @@ public class Block {
/** Remote/imported/loaded AT states */
protected List<ATStateData> atStates;
/** Remote hash of AT states - in lieu of full AT state data in {@code atStates} */
protected byte[] atStatesHash;
/** Locally-generated AT states */
protected List<ATStateData> ourAtStates;
/** Locally-generated AT fees */
@ -255,7 +260,7 @@ public class Block {
* Constructs new Block using passed transaction and AT states.
* <p>
* This constructor typically used when receiving a serialized block over the network.
*
*
* @param repository
* @param blockData
* @param transactions
@ -281,6 +286,35 @@ public class Block {
this.blockData.setTotalFees(totalFees);
}
/**
* Constructs new Block using passed transaction and minimal AT state info.
* <p>
* This constructor typically used when receiving a serialized block over the network.
*
* @param repository
* @param blockData
* @param transactions
* @param atStatesHash
*/
public Block(Repository repository, BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
this(repository, blockData);
this.transactions = new ArrayList<>();
long totalFees = 0;
// We have to sum fees too
for (TransactionData transactionData : transactions) {
this.transactions.add(Transaction.fromData(repository, transactionData));
totalFees += transactionData.getFee();
}
this.atStatesHash = atStatesHash;
totalFees += this.blockData.getATFees();
this.blockData.setTotalFees(totalFees);
}
/**
* Constructs new Block with empty transaction list, using passed minter account.
*
@ -1194,7 +1228,7 @@ public class Block {
*/
private ValidationResult areAtsValid() throws DataException {
// Locally generated AT states should be valid so no need to re-execute them
if (this.ourAtStates == this.getATStates()) // Note object reference compare
if (this.ourAtStates != null && this.ourAtStates == this.atStates) // Note object reference compare
return ValidationResult.OK;
// Generate local AT states for comparison
@ -1208,8 +1242,33 @@ public class Block {
if (this.ourAtFees != this.blockData.getATFees())
return ValidationResult.AT_STATES_MISMATCH;
// Note: this.atStates fully loaded thanks to this.getATStates() call above
for (int s = 0; s < this.atStates.size(); ++s) {
// If we have a single AT states hash then compare that in preference
if (this.atStatesHash != null) {
int atBytesLength = blockData.getATCount() * BlockTransformer.AT_ENTRY_LENGTH;
ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength);
try {
for (ATStateData atStateData : this.ourAtStates) {
atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
atHashBytes.write(atStateData.getStateHash());
atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
}
} catch (IOException e) {
throw new DataException("Couldn't validate AT states hash due to serialization issue?", e);
}
byte[] ourAtStatesHash = Crypto.digest(atHashBytes.toByteArray());
if (!Arrays.equals(ourAtStatesHash, this.atStatesHash))
return ValidationResult.AT_STATES_MISMATCH;
// Use our AT state data from now on
this.atStates = this.ourAtStates;
return ValidationResult.OK;
}
// Note: this.atStates fully loaded thanks to this.getATStates() call:
this.getATStates();
for (int s = 0; s < this.ourAtStates.size(); ++s) {
ATStateData ourAtState = this.ourAtStates.get(s);
ATStateData theirAtState = this.atStates.get(s);

12
src/main/java/org/qortal/controller/Controller.java
View File

@ -1362,6 +1362,18 @@ public class Controller extends Thread {
Block block = new Block(repository, blockData);
// V2 support
if (peer.getPeersVersion() >= BlockV2Message.MIN_PEER_VERSION) {
Message blockMessage = new BlockV2Message(block);
blockMessage.setId(message.getId());
if (!peer.sendMessage(blockMessage)) {
peer.disconnect("failed to send block");
// Don't fall-through to caching because failure to send might be from failure to build message
return;
}
return;
}
CachedBlockMessage blockMessage = new CachedBlockMessage(block);
blockMessage.setId(message.getId());

26
src/main/java/org/qortal/controller/Synchronizer.java
View File

@ -26,14 +26,7 @@ import org.qortal.event.Event;
import org.qortal.event.EventBus;
import org.qortal.network.Network;
import org.qortal.network.Peer;
import org.qortal.network.message.BlockMessage;
import org.qortal.network.message.BlockSummariesMessage;
import org.qortal.network.message.GetBlockMessage;
import org.qortal.network.message.GetBlockSummariesMessage;
import org.qortal.network.message.GetSignaturesV2Message;
import org.qortal.network.message.Message;
import org.qortal.network.message.SignaturesMessage;
import org.qortal.network.message.MessageType;
import org.qortal.network.message.*;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager;
@ -1579,12 +1572,23 @@ public class Synchronizer extends Thread {
Message getBlockMessage = new GetBlockMessage(signature);
Message message = peer.getResponse(getBlockMessage);
if (message == null || message.getType() != MessageType.BLOCK)
if (message == null)
return null;
BlockMessage blockMessage = (BlockMessage) message;
switch (message.getType()) {
case BLOCK: {
BlockMessage blockMessage = (BlockMessage) message;
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
}
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
case BLOCK_V2: {
BlockV2Message blockMessage = (BlockV2Message) message;
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStatesHash());
}
default:
return null;
}
}
public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {

7
src/main/java/org/qortal/network/message/BlockMessage.java
View File

@ -9,6 +9,7 @@ import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import org.qortal.utils.Triple;
@ -46,12 +47,12 @@ public class BlockMessage extends Message {
try {
int height = byteBuffer.getInt();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
BlockTransformation blockTransformation = BlockTransformer.fromByteBuffer(byteBuffer);
BlockData blockData = blockInfo.getA();
BlockData blockData = blockTransformation.getBlockData();
blockData.setHeight(height);
return new BlockMessage(id, blockData, blockInfo.getB(), blockInfo.getC());
return new BlockMessage(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStates());
} catch (TransformationException e) {
LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage()));
throw new MessageException(e.getMessage(), e);

87
src/main/java/org/qortal/network/message/BlockV2Message.java Normal file
View File

@ -0,0 +1,87 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.Block;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
public class BlockV2Message extends Message {
private static final Logger LOGGER = LogManager.getLogger(BlockV2Message.class);
public static final long MIN_PEER_VERSION = 0x3000300cbL; // 3.3.203
private BlockData blockData;
private List<TransactionData> transactions;
private byte[] atStatesHash;
public BlockV2Message(Block block) throws TransformationException {
super(MessageType.BLOCK_V2);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
bytes.write(BlockTransformer.toBytesV2(block));
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
public BlockV2Message(byte[] cachedBytes) {
super(MessageType.BLOCK_V2);
this.dataBytes = cachedBytes;
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private BlockV2Message(int id, BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
super(id, MessageType.BLOCK_V2);
this.blockData = blockData;
this.transactions = transactions;
this.atStatesHash = atStatesHash;
}
public BlockData getBlockData() {
return this.blockData;
}
public List<TransactionData> getTransactions() {
return this.transactions;
}
public byte[] getAtStatesHash() {
return this.atStatesHash;
}
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException {
try {
int height = byteBuffer.getInt();
BlockTransformation blockTransformation = BlockTransformer.fromByteBufferV2(byteBuffer);
BlockData blockData = blockTransformation.getBlockData();
blockData.setHeight(height);
return new BlockV2Message(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStatesHash());
} catch (TransformationException e) {
LOGGER.info(String.format("Received garbled BLOCK_V2 message: %s", e.getMessage()));
throw new MessageException(e.getMessage(), e);
}
}
}

1
src/main/java/org/qortal/network/message/MessageType.java
View File

@ -34,6 +34,7 @@ public enum MessageType {
BLOCK(50, BlockMessage::fromByteBuffer),
GET_BLOCK(51, GetBlockMessage::fromByteBuffer),
BLOCK_V2(52, BlockV2Message::fromByteBuffer),
SIGNATURES(60, SignaturesMessage::fromByteBuffer),
GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer),

19
src/main/java/org/qortal/repository/BlockArchiveReader.java
View File

@ -9,6 +9,7 @@ import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.settings.Settings;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import org.qortal.utils.Triple;
@ -66,7 +67,7 @@ public class BlockArchiveReader {
this.fileListCache = Map.copyOf(map);
}
public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockAtHeight(int height) {
public BlockTransformation fetchBlockAtHeight(int height) {
if (this.fileListCache == null) {
this.fetchFileList();
}
@ -77,13 +78,13 @@ public class BlockArchiveReader {
}
ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes);
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = null;
BlockTransformation blockInfo = null;
try {
blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
if (blockInfo != null && blockInfo.getA() != null) {
if (blockInfo != null && blockInfo.getBlockData() != null) {
// Block height is stored outside of the main serialized bytes, so it
// won't be set automatically.
blockInfo.getA().setHeight(height);
blockInfo.getBlockData().setHeight(height);
}
} catch (TransformationException e) {
return null;
@ -91,8 +92,7 @@ public class BlockArchiveReader {
return blockInfo;
}
public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockWithSignature(
byte[] signature, Repository repository) {
public BlockTransformation fetchBlockWithSignature(byte[] signature, Repository repository) {
if (this.fileListCache == null) {
this.fetchFileList();
@ -105,13 +105,12 @@ public class BlockArchiveReader {
return null;
}
public List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> fetchBlocksFromRange(
int startHeight, int endHeight) {
public List<BlockTransformation> fetchBlocksFromRange(int startHeight, int endHeight) {
List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> blockInfoList = new ArrayList<>();
List<BlockTransformation> blockInfoList = new ArrayList<>();
for (int height = startHeight; height <= endHeight; height++) {
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = this.fetchBlockAtHeight(height);
BlockTransformation blockInfo = this.fetchBlockAtHeight(height);
if (blockInfo == null) {
return blockInfoList;
}

29
src/main/java/org/qortal/repository/hsqldb/HSQLDBBlockArchiveRepository.java
View File

@ -1,16 +1,13 @@
package org.qortal.repository.hsqldb;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.model.BlockSignerSummary;
import org.qortal.block.Block;
import org.qortal.data.block.BlockArchiveData;
import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData;
import org.qortal.repository.BlockArchiveReader;
import org.qortal.repository.BlockArchiveRepository;
import org.qortal.repository.DataException;
import org.qortal.utils.Triple;
import org.qortal.transform.block.BlockTransformation;
import java.sql.ResultSet;
import java.sql.SQLException;
@ -29,11 +26,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
@Override
public BlockData fromSignature(byte[] signature) throws DataException {
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository);
if (blockInfo != null) {
return (BlockData) blockInfo.getA();
}
return null;
BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository);
if (blockInfo == null)
return null;
return blockInfo.getBlockData();
}
@Override
@ -47,11 +44,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
@Override
public BlockData fromHeight(int height) throws DataException {
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
if (blockInfo != null) {
return (BlockData) blockInfo.getA();
}
return null;
BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
if (blockInfo == null)
return null;
return blockInfo.getBlockData();
}
@Override
@ -79,9 +76,9 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
int height = referenceBlock.getHeight();
if (height > 0) {
// Request the block at height + 1
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1);
BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1);
if (blockInfo != null) {
return (BlockData) blockInfo.getA();
return blockInfo.getBlockData();
}
}
}

44
src/main/java/org/qortal/transform/block/BlockTransformation.java Normal file
View File

@ -0,0 +1,44 @@
package org.qortal.transform.block;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import java.util.List;
public class BlockTransformation {
private final BlockData blockData;
private final List<TransactionData> transactions;
private final List<ATStateData> atStates;
private final byte[] atStatesHash;
/*package*/ BlockTransformation(BlockData blockData, List<TransactionData> transactions, List<ATStateData> atStates) {
this.blockData = blockData;
this.transactions = transactions;
this.atStates = atStates;
this.atStatesHash = null;
}
/*package*/ BlockTransformation(BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
this.blockData = blockData;
this.transactions = transactions;
this.atStates = null;
this.atStatesHash = atStatesHash;
}
public BlockData getBlockData() {
return blockData;
}
public List<TransactionData> getTransactions() {
return transactions;
}
public List<ATStateData> getAtStates() {
return atStates;
}
public byte[] getAtStatesHash() {
return atStatesHash;
}
}

150
src/main/java/org/qortal/transform/block/BlockTransformer.java
View File

@ -3,12 +3,14 @@ package org.qortal.transform.block;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.qortal.block.Block;
import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
@ -20,7 +22,6 @@ import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58;
import org.qortal.utils.Serialization;
import org.qortal.utils.Triple;
import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs;
@ -45,14 +46,13 @@ public class BlockTransformer extends Transformer {
protected static final int AT_BYTES_LENGTH = INT_LENGTH;
protected static final int AT_FEES_LENGTH = AMOUNT_LENGTH;
protected static final int AT_LENGTH = AT_FEES_LENGTH + AT_BYTES_LENGTH;
protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = TIMESTAMP_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH;
protected static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH;
public static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH;
/**
* Extract block data and transaction data from serialized bytes.
@ -61,7 +61,7 @@ public class BlockTransformer extends Transformer {
* @return BlockData and a List of transactions.
* @throws TransformationException
*/
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromBytes(byte[] bytes) throws TransformationException {
public static BlockTransformation fromBytes(byte[] bytes) throws TransformationException {
if (bytes == null)
return null;
@ -76,28 +76,40 @@ public class BlockTransformer extends Transformer {
/**
* Extract block data and transaction data from serialized bytes containing a single block.
*
* @param bytes
* @param byteBuffer source of serialized block bytes
* @return BlockData and a List of transactions.
* @throws TransformationException
*/
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException {
public static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException {
return BlockTransformer.fromByteBuffer(byteBuffer, false);
}
/**
* Extract block data and transaction data from serialized bytes containing a single block.
*
* @param byteBuffer source of serialized block bytes
* @return BlockData and a List of transactions.
* @throws TransformationException
*/
public static BlockTransformation fromByteBufferV2(ByteBuffer byteBuffer) throws TransformationException {
return BlockTransformer.fromByteBuffer(byteBuffer, true);
}
/**
* Extract block data and transaction data from serialized bytes containing one or more blocks.
*
* @param bytes
* Extract block data and transaction data from serialized bytes containing a single block, in one of two forms.
*
* @param byteBuffer source of serialized block bytes
* @param isV2 set to true if AT state info is represented by a single hash, false if serialized as per-AT address+state hash+fees
* @return the next block's BlockData and a List of transactions.
* @throws TransformationException
*/
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromByteBuffer(ByteBuffer byteBuffer, boolean finalBlockInBuffer) throws TransformationException {
private static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer, boolean isV2) throws TransformationException {
int version = byteBuffer.getInt();
if (finalBlockInBuffer && byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH)
if (byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH)
throw new TransformationException("Byte data too short for Block");
if (finalBlockInBuffer && byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize())
if (byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for Block");
long timestamp = byteBuffer.getLong();
@ -117,42 +129,52 @@ public class BlockTransformer extends Transformer {
int atCount = 0;
long atFees = 0;
List<ATStateData> atStates = new ArrayList<>();
byte[] atStatesHash = null;
List<ATStateData> atStates = null;
int atBytesLength = byteBuffer.getInt();
if (isV2) {
// Simply: AT count, AT total fees, hash(all AT states)
atCount = byteBuffer.getInt();
atFees = byteBuffer.getLong();
atStatesHash = new byte[Transformer.SHA256_LENGTH];
byteBuffer.get(atStatesHash);
} else {
// V1: AT info byte length, then per-AT entries of AT address + state hash + fees
int atBytesLength = byteBuffer.getInt();
if (atBytesLength > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for Block's AT info");
if (atBytesLength > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for Block's AT info");
// Read AT-address, SHA256 hash and fees
if (atBytesLength % AT_ENTRY_LENGTH != 0)
throw new TransformationException("AT byte data not a multiple of AT entry length");
ByteBuffer atByteBuffer = byteBuffer.slice();
atByteBuffer.limit(atBytesLength);
ByteBuffer atByteBuffer = byteBuffer.slice();
atByteBuffer.limit(atBytesLength);
// Read AT-address, SHA256 hash and fees
if (atBytesLength % AT_ENTRY_LENGTH != 0)
throw new TransformationException("AT byte data not a multiple of AT entry length");
atStates = new ArrayList<>();
while (atByteBuffer.hasRemaining()) {
byte[] atAddressBytes = new byte[ADDRESS_LENGTH];
atByteBuffer.get(atAddressBytes);
String atAddress = Base58.encode(atAddressBytes);
while (atByteBuffer.hasRemaining()) {
byte[] atAddressBytes = new byte[ADDRESS_LENGTH];
atByteBuffer.get(atAddressBytes);
String atAddress = Base58.encode(atAddressBytes);
byte[] stateHash = new byte[SHA256_LENGTH];
atByteBuffer.get(stateHash);
byte[] stateHash = new byte[SHA256_LENGTH];
atByteBuffer.get(stateHash);
long fees = atByteBuffer.getLong();
long fees = atByteBuffer.getLong();
// Add this AT's fees to our total
atFees += fees;
// Add this AT's fees to our total
atFees += fees;
atStates.add(new ATStateData(atAddress, stateHash, fees));
}
atStates.add(new ATStateData(atAddress, stateHash, fees));
// Bump byteBuffer over AT states just read in slice
byteBuffer.position(byteBuffer.position() + atBytesLength);
// AT count to reflect the number of states we have
atCount = atStates.size();
}
// Bump byteBuffer over AT states just read in slice
byteBuffer.position(byteBuffer.position() + atBytesLength);
// AT count to reflect the number of states we have
atCount = atStates.size();
// Add AT fees to totalFees
totalFees += atFees;
@ -221,16 +243,15 @@ public class BlockTransformer extends Transformer {
byteBuffer.get(onlineAccountsSignatures);
}
// We should only complain about excess byte data if we aren't expecting more blocks in this ByteBuffer
if (finalBlockInBuffer && byteBuffer.hasRemaining())
throw new TransformationException("Excess byte data found after parsing Block");
// We don't have a height!
Integer height = null;
BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp,
minterPublicKey, minterSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
return new Triple<>(blockData, transactions, atStates);
if (isV2)
return new BlockTransformation(blockData, transactions, atStatesHash);
else
return new BlockTransformation(blockData, transactions, atStates);
}
public static int getDataLength(Block block) throws TransformationException {
@ -266,6 +287,14 @@ public class BlockTransformer extends Transformer {
}
public static byte[] toBytes(Block block) throws TransformationException {
return toBytes(block, false);
}
public static byte[] toBytesV2(Block block) throws TransformationException {
return toBytes(block, true);
}
private static byte[] toBytes(Block block, boolean isV2) throws TransformationException {
BlockData blockData = block.getBlockData();
try {
@ -279,16 +308,37 @@ public class BlockTransformer extends Transformer {
bytes.write(blockData.getMinterSignature());
int atBytesLength = blockData.getATCount() * AT_ENTRY_LENGTH;
bytes.write(Ints.toByteArray(atBytesLength));
if (isV2) {
ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength);
long atFees = 0;
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
bytes.write(Base58.decode(atStateData.getATAddress()));
bytes.write(atStateData.getStateHash());
bytes.write(Longs.toByteArray(atStateData.getFees()));
atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
atHashBytes.write(atStateData.getStateHash());
atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
atFees += atStateData.getFees();
}
bytes.write(Ints.toByteArray(blockData.getATCount()));
bytes.write(Longs.toByteArray(atFees));
bytes.write(Crypto.digest(atHashBytes.toByteArray()));
} else {
bytes.write(Ints.toByteArray(atBytesLength));
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
bytes.write(Base58.decode(atStateData.getATAddress()));
bytes.write(atStateData.getStateHash());
bytes.write(Longs.toByteArray(atStateData.getFees()));
}
}
// Transactions

21
src/main/java/org/qortal/utils/BlockArchiveUtils.java
View File

@ -6,6 +6,7 @@ import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.BlockArchiveReader;
import org.qortal.repository.DataException;
import org.qortal.repository.Repository;
import org.qortal.transform.block.BlockTransformation;
import java.util.List;
@ -33,8 +34,7 @@ public class BlockArchiveUtils {
repository.discardChanges();
final int requestedRange = endHeight+1-startHeight;
List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> blockInfoList =
BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight);
List<BlockTransformation> blockInfoList = BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight);
// Ensure that we have received all of the requested blocks
if (blockInfoList == null || blockInfoList.isEmpty()) {
@ -43,27 +43,26 @@ public class BlockArchiveUtils {
if (blockInfoList.size() != requestedRange) {
throw new IllegalStateException("Non matching block count when importing from archive");
}
Triple<BlockData, List<TransactionData>, List<ATStateData>> firstBlock = blockInfoList.get(0);
if (firstBlock == null || firstBlock.getA().getHeight() != startHeight) {
BlockTransformation firstBlock = blockInfoList.get(0);
if (firstBlock == null || firstBlock.getBlockData().getHeight() != startHeight) {
throw new IllegalStateException("Non matching first block when importing from archive");
}
if (blockInfoList.size() > 0) {
Triple<BlockData, List<TransactionData>, List<ATStateData>> lastBlock =
blockInfoList.get(blockInfoList.size() - 1);
if (lastBlock == null || lastBlock.getA().getHeight() != endHeight) {
BlockTransformation lastBlock = blockInfoList.get(blockInfoList.size() - 1);
if (lastBlock == null || lastBlock.getBlockData().getHeight() != endHeight) {
throw new IllegalStateException("Non matching last block when importing from archive");
}
}
// Everything seems okay, so go ahead with the import
for (Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo : blockInfoList) {
for (BlockTransformation blockInfo : blockInfoList) {
try {
// Save block
repository.getBlockRepository().save(blockInfo.getA());
repository.getBlockRepository().save(blockInfo.getBlockData());
// Save AT state data hashes
for (ATStateData atStateData : blockInfo.getC()) {
atStateData.setHeight(blockInfo.getA().getHeight());
for (ATStateData atStateData : blockInfo.getAtStates()) {
atStateData.setHeight(blockInfo.getBlockData().getHeight());
repository.getATRepository().save(atStateData);
}

19
src/test/java/org/qortal/test/BlockArchiveTests.java
View File

@ -20,6 +20,7 @@ import org.qortal.test.common.Common;
import org.qortal.transaction.DeployAtTransaction;
import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.utils.BlockArchiveUtils;
import org.qortal.utils.NTP;
import org.qortal.utils.Triple;
@ -123,8 +124,8 @@ public class BlockArchiveTests extends Common {
// Read block 2 from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> block2Info = reader.fetchBlockAtHeight(2);
BlockData block2ArchiveData = block2Info.getA();
BlockTransformation block2Info = reader.fetchBlockAtHeight(2);
BlockData block2ArchiveData = block2Info.getBlockData();
// Read block 2 from the repository
BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2);
@ -137,8 +138,8 @@ public class BlockArchiveTests extends Common {
assertEquals(1, block2ArchiveData.getOnlineAccountsCount());
// Read block 900 from the archive
Triple<BlockData, List<TransactionData>, List<ATStateData>> block900Info = reader.fetchBlockAtHeight(900);
BlockData block900ArchiveData = block900Info.getA();
BlockTransformation block900Info = reader.fetchBlockAtHeight(900);
BlockData block900ArchiveData = block900Info.getBlockData();
// Read block 900 from the repository
BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900);
@ -200,10 +201,10 @@ public class BlockArchiveTests extends Common {
// Read a block from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(testHeight);
BlockData archivedBlockData = blockInfo.getA();
ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0);
List<TransactionData> archivedTransactions = blockInfo.getB();
BlockTransformation blockInfo = reader.fetchBlockAtHeight(testHeight);
BlockData archivedBlockData = blockInfo.getBlockData();
ATStateData archivedAtStateData = blockInfo.getAtStates().isEmpty() ? null : blockInfo.getAtStates().get(0);
List<TransactionData> archivedTransactions = blockInfo.getTransactions();
// Read the same block from the repository
BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight);
@ -255,7 +256,7 @@ public class BlockArchiveTests extends Common {
// Check block 10 (unarchived)
BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(10);
BlockTransformation blockInfo = reader.fetchBlockAtHeight(10);
assertNull(blockInfo);
}

5
src/test/java/org/qortal/test/BlockTests.java
View File

@ -24,6 +24,7 @@ import org.qortal.test.common.TransactionUtils;
import org.qortal.transaction.Transaction;
import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58;
@ -121,10 +122,10 @@ public class BlockTests extends Common {
assertEquals(BlockTransformer.getDataLength(block), bytes.length);
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = BlockTransformer.fromBytes(bytes);
BlockTransformation blockInfo = BlockTransformer.fromBytes(bytes);
// Compare transactions
List<TransactionData> deserializedTransactions = blockInfo.getB();
List<TransactionData> deserializedTransactions = blockInfo.getTransactions();
assertEquals("Transaction count differs", blockData.getTransactionCount(), deserializedTransactions.size());
for (int i = 0; i < blockData.getTransactionCount(); ++i) {