Initial work on reducing network load for transferring blocks.

Reduced AT state info from per-AT address + state hash + fees to AT count + total AT fees + hash of all AT states.
Modified Block and Controller to support above. Controller needs more work regarding CachedBlockMessages.
Note that blocks fetched from archive are in old V1 format.
Changed Triple<BlockData, List<TransactionData>, List<ATStateData>> to BlockTransformation to support both V1 and V2 forms.

Set min peer version to 3.3.203 in BlockV2Message class.
This commit is contained in:
catbref 2022-04-17 19:22:29 +01:00 committed by CalDescent
parent ba7b9f3ad8
commit e85026f866
13 changed files with 371 additions and 116 deletions

View File

@ -3,9 +3,12 @@ package org.qortal.block;
import static java.util.Arrays.stream; import static java.util.Arrays.stream;
import static java.util.stream.Collectors.toMap; import static java.util.stream.Collectors.toMap;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
import java.math.RoundingMode; import java.math.RoundingMode;
import java.nio.charset.StandardCharsets;
import java.text.DecimalFormat; import java.text.DecimalFormat;
import java.text.NumberFormat; import java.text.NumberFormat;
import java.util.*; import java.util.*;
@ -118,6 +121,8 @@ public class Block {
/** Remote/imported/loaded AT states */ /** Remote/imported/loaded AT states */
protected List<ATStateData> atStates; protected List<ATStateData> atStates;
/** Remote hash of AT states - in lieu of full AT state data in {@code atStates} */
protected byte[] atStatesHash;
/** Locally-generated AT states */ /** Locally-generated AT states */
protected List<ATStateData> ourAtStates; protected List<ATStateData> ourAtStates;
/** Locally-generated AT fees */ /** Locally-generated AT fees */
@ -255,7 +260,7 @@ public class Block {
* Constructs new Block using passed transaction and AT states. * Constructs new Block using passed transaction and AT states.
* <p> * <p>
* This constructor typically used when receiving a serialized block over the network. * This constructor typically used when receiving a serialized block over the network.
* *
* @param repository * @param repository
* @param blockData * @param blockData
* @param transactions * @param transactions
@ -281,6 +286,35 @@ public class Block {
this.blockData.setTotalFees(totalFees); this.blockData.setTotalFees(totalFees);
} }
/**
* Constructs new Block using passed transaction and minimal AT state info.
* <p>
* This constructor typically used when receiving a serialized block over the network.
*
* @param repository
* @param blockData
* @param transactions
* @param atStatesHash
*/
public Block(Repository repository, BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
this(repository, blockData);
this.transactions = new ArrayList<>();
long totalFees = 0;
// We have to sum fees too
for (TransactionData transactionData : transactions) {
this.transactions.add(Transaction.fromData(repository, transactionData));
totalFees += transactionData.getFee();
}
this.atStatesHash = atStatesHash;
totalFees += this.blockData.getATFees();
this.blockData.setTotalFees(totalFees);
}
/** /**
* Constructs new Block with empty transaction list, using passed minter account. * Constructs new Block with empty transaction list, using passed minter account.
* *
@ -1194,7 +1228,7 @@ public class Block {
*/ */
private ValidationResult areAtsValid() throws DataException { private ValidationResult areAtsValid() throws DataException {
// Locally generated AT states should be valid so no need to re-execute them // Locally generated AT states should be valid so no need to re-execute them
if (this.ourAtStates == this.getATStates()) // Note object reference compare if (this.ourAtStates != null && this.ourAtStates == this.atStates) // Note object reference compare
return ValidationResult.OK; return ValidationResult.OK;
// Generate local AT states for comparison // Generate local AT states for comparison
@ -1208,8 +1242,33 @@ public class Block {
if (this.ourAtFees != this.blockData.getATFees()) if (this.ourAtFees != this.blockData.getATFees())
return ValidationResult.AT_STATES_MISMATCH; return ValidationResult.AT_STATES_MISMATCH;
// Note: this.atStates fully loaded thanks to this.getATStates() call above // If we have a single AT states hash then compare that in preference
for (int s = 0; s < this.atStates.size(); ++s) { if (this.atStatesHash != null) {
int atBytesLength = blockData.getATCount() * BlockTransformer.AT_ENTRY_LENGTH;
ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength);
try {
for (ATStateData atStateData : this.ourAtStates) {
atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
atHashBytes.write(atStateData.getStateHash());
atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
}
} catch (IOException e) {
throw new DataException("Couldn't validate AT states hash due to serialization issue?", e);
}
byte[] ourAtStatesHash = Crypto.digest(atHashBytes.toByteArray());
if (!Arrays.equals(ourAtStatesHash, this.atStatesHash))
return ValidationResult.AT_STATES_MISMATCH;
// Use our AT state data from now on
this.atStates = this.ourAtStates;
return ValidationResult.OK;
}
// Note: this.atStates fully loaded thanks to this.getATStates() call:
this.getATStates();
for (int s = 0; s < this.ourAtStates.size(); ++s) {
ATStateData ourAtState = this.ourAtStates.get(s); ATStateData ourAtState = this.ourAtStates.get(s);
ATStateData theirAtState = this.atStates.get(s); ATStateData theirAtState = this.atStates.get(s);

View File

@ -1362,6 +1362,18 @@ public class Controller extends Thread {
Block block = new Block(repository, blockData); Block block = new Block(repository, blockData);
// V2 support
if (peer.getPeersVersion() >= BlockV2Message.MIN_PEER_VERSION) {
Message blockMessage = new BlockV2Message(block);
blockMessage.setId(message.getId());
if (!peer.sendMessage(blockMessage)) {
peer.disconnect("failed to send block");
// Don't fall-through to caching because failure to send might be from failure to build message
return;
}
return;
}
CachedBlockMessage blockMessage = new CachedBlockMessage(block); CachedBlockMessage blockMessage = new CachedBlockMessage(block);
blockMessage.setId(message.getId()); blockMessage.setId(message.getId());

View File

@ -26,14 +26,7 @@ import org.qortal.event.Event;
import org.qortal.event.EventBus; import org.qortal.event.EventBus;
import org.qortal.network.Network; import org.qortal.network.Network;
import org.qortal.network.Peer; import org.qortal.network.Peer;
import org.qortal.network.message.BlockMessage; import org.qortal.network.message.*;
import org.qortal.network.message.BlockSummariesMessage;
import org.qortal.network.message.GetBlockMessage;
import org.qortal.network.message.GetBlockSummariesMessage;
import org.qortal.network.message.GetSignaturesV2Message;
import org.qortal.network.message.Message;
import org.qortal.network.message.SignaturesMessage;
import org.qortal.network.message.MessageType;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.repository.RepositoryManager; import org.qortal.repository.RepositoryManager;
@ -1579,12 +1572,23 @@ public class Synchronizer extends Thread {
Message getBlockMessage = new GetBlockMessage(signature); Message getBlockMessage = new GetBlockMessage(signature);
Message message = peer.getResponse(getBlockMessage); Message message = peer.getResponse(getBlockMessage);
if (message == null || message.getType() != MessageType.BLOCK) if (message == null)
return null; return null;
BlockMessage blockMessage = (BlockMessage) message; switch (message.getType()) {
case BLOCK: {
BlockMessage blockMessage = (BlockMessage) message;
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates());
}
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStates()); case BLOCK_V2: {
BlockV2Message blockMessage = (BlockV2Message) message;
return new Block(repository, blockMessage.getBlockData(), blockMessage.getTransactions(), blockMessage.getAtStatesHash());
}
default:
return null;
}
} }
public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException { public void populateBlockSummariesMinterLevels(Repository repository, List<BlockSummaryData> blockSummaries) throws DataException {

View File

@ -9,6 +9,7 @@ import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
import org.qortal.transform.TransformationException; import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer; import org.qortal.transform.block.BlockTransformer;
import org.qortal.utils.Triple; import org.qortal.utils.Triple;
@ -46,12 +47,12 @@ public class BlockMessage extends Message {
try { try {
int height = byteBuffer.getInt(); int height = byteBuffer.getInt();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); BlockTransformation blockTransformation = BlockTransformer.fromByteBuffer(byteBuffer);
BlockData blockData = blockInfo.getA(); BlockData blockData = blockTransformation.getBlockData();
blockData.setHeight(height); blockData.setHeight(height);
return new BlockMessage(id, blockData, blockInfo.getB(), blockInfo.getC()); return new BlockMessage(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStates());
} catch (TransformationException e) { } catch (TransformationException e) {
LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage())); LOGGER.info(String.format("Received garbled BLOCK message: %s", e.getMessage()));
throw new MessageException(e.getMessage(), e); throw new MessageException(e.getMessage(), e);

View File

@ -0,0 +1,87 @@
package org.qortal.network.message;
import com.google.common.primitives.Ints;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.qortal.block.Block;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.List;
public class BlockV2Message extends Message {
private static final Logger LOGGER = LogManager.getLogger(BlockV2Message.class);
public static final long MIN_PEER_VERSION = 0x3000300cbL; // 3.3.203
private BlockData blockData;
private List<TransactionData> transactions;
private byte[] atStatesHash;
public BlockV2Message(Block block) throws TransformationException {
super(MessageType.BLOCK_V2);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
try {
bytes.write(Ints.toByteArray(block.getBlockData().getHeight()));
bytes.write(BlockTransformer.toBytesV2(block));
} catch (IOException e) {
throw new AssertionError("IOException shouldn't occur with ByteArrayOutputStream");
}
this.dataBytes = bytes.toByteArray();
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
public BlockV2Message(byte[] cachedBytes) {
super(MessageType.BLOCK_V2);
this.dataBytes = cachedBytes;
this.checksumBytes = Message.generateChecksum(this.dataBytes);
}
private BlockV2Message(int id, BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
super(id, MessageType.BLOCK_V2);
this.blockData = blockData;
this.transactions = transactions;
this.atStatesHash = atStatesHash;
}
public BlockData getBlockData() {
return this.blockData;
}
public List<TransactionData> getTransactions() {
return this.transactions;
}
public byte[] getAtStatesHash() {
return this.atStatesHash;
}
public static Message fromByteBuffer(int id, ByteBuffer byteBuffer) throws MessageException {
try {
int height = byteBuffer.getInt();
BlockTransformation blockTransformation = BlockTransformer.fromByteBufferV2(byteBuffer);
BlockData blockData = blockTransformation.getBlockData();
blockData.setHeight(height);
return new BlockV2Message(id, blockData, blockTransformation.getTransactions(), blockTransformation.getAtStatesHash());
} catch (TransformationException e) {
LOGGER.info(String.format("Received garbled BLOCK_V2 message: %s", e.getMessage()));
throw new MessageException(e.getMessage(), e);
}
}
}

View File

@ -34,6 +34,7 @@ public enum MessageType {
BLOCK(50, BlockMessage::fromByteBuffer), BLOCK(50, BlockMessage::fromByteBuffer),
GET_BLOCK(51, GetBlockMessage::fromByteBuffer), GET_BLOCK(51, GetBlockMessage::fromByteBuffer),
BLOCK_V2(52, BlockV2Message::fromByteBuffer),
SIGNATURES(60, SignaturesMessage::fromByteBuffer), SIGNATURES(60, SignaturesMessage::fromByteBuffer),
GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer), GET_SIGNATURES_V2(61, GetSignaturesV2Message::fromByteBuffer),

View File

@ -9,6 +9,7 @@ import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
import org.qortal.settings.Settings; import org.qortal.settings.Settings;
import org.qortal.transform.TransformationException; import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer; import org.qortal.transform.block.BlockTransformer;
import org.qortal.utils.Triple; import org.qortal.utils.Triple;
@ -66,7 +67,7 @@ public class BlockArchiveReader {
this.fileListCache = Map.copyOf(map); this.fileListCache = Map.copyOf(map);
} }
public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockAtHeight(int height) { public BlockTransformation fetchBlockAtHeight(int height) {
if (this.fileListCache == null) { if (this.fileListCache == null) {
this.fetchFileList(); this.fetchFileList();
} }
@ -77,13 +78,13 @@ public class BlockArchiveReader {
} }
ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes); ByteBuffer byteBuffer = ByteBuffer.wrap(serializedBytes);
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = null; BlockTransformation blockInfo = null;
try { try {
blockInfo = BlockTransformer.fromByteBuffer(byteBuffer); blockInfo = BlockTransformer.fromByteBuffer(byteBuffer);
if (blockInfo != null && blockInfo.getA() != null) { if (blockInfo != null && blockInfo.getBlockData() != null) {
// Block height is stored outside of the main serialized bytes, so it // Block height is stored outside of the main serialized bytes, so it
// won't be set automatically. // won't be set automatically.
blockInfo.getA().setHeight(height); blockInfo.getBlockData().setHeight(height);
} }
} catch (TransformationException e) { } catch (TransformationException e) {
return null; return null;
@ -91,8 +92,7 @@ public class BlockArchiveReader {
return blockInfo; return blockInfo;
} }
public Triple<BlockData, List<TransactionData>, List<ATStateData>> fetchBlockWithSignature( public BlockTransformation fetchBlockWithSignature(byte[] signature, Repository repository) {
byte[] signature, Repository repository) {
if (this.fileListCache == null) { if (this.fileListCache == null) {
this.fetchFileList(); this.fetchFileList();
@ -105,13 +105,12 @@ public class BlockArchiveReader {
return null; return null;
} }
public List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> fetchBlocksFromRange( public List<BlockTransformation> fetchBlocksFromRange(int startHeight, int endHeight) {
int startHeight, int endHeight) {
List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> blockInfoList = new ArrayList<>(); List<BlockTransformation> blockInfoList = new ArrayList<>();
for (int height = startHeight; height <= endHeight; height++) { for (int height = startHeight; height <= endHeight; height++) {
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = this.fetchBlockAtHeight(height); BlockTransformation blockInfo = this.fetchBlockAtHeight(height);
if (blockInfo == null) { if (blockInfo == null) {
return blockInfoList; return blockInfoList;
} }

View File

@ -1,16 +1,13 @@
package org.qortal.repository.hsqldb; package org.qortal.repository.hsqldb;
import org.qortal.api.ApiError;
import org.qortal.api.ApiExceptionFactory;
import org.qortal.api.model.BlockSignerSummary; import org.qortal.api.model.BlockSignerSummary;
import org.qortal.block.Block;
import org.qortal.data.block.BlockArchiveData; import org.qortal.data.block.BlockArchiveData;
import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockData;
import org.qortal.data.block.BlockSummaryData; import org.qortal.data.block.BlockSummaryData;
import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.BlockArchiveReader;
import org.qortal.repository.BlockArchiveRepository; import org.qortal.repository.BlockArchiveRepository;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.utils.Triple; import org.qortal.transform.block.BlockTransformation;
import java.sql.ResultSet; import java.sql.ResultSet;
import java.sql.SQLException; import java.sql.SQLException;
@ -29,11 +26,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
@Override @Override
public BlockData fromSignature(byte[] signature) throws DataException { public BlockData fromSignature(byte[] signature) throws DataException {
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository); BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockWithSignature(signature, this.repository);
if (blockInfo != null) { if (blockInfo == null)
return (BlockData) blockInfo.getA(); return null;
}
return null; return blockInfo.getBlockData();
} }
@Override @Override
@ -47,11 +44,11 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
@Override @Override
public BlockData fromHeight(int height) throws DataException { public BlockData fromHeight(int height) throws DataException {
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height); BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height);
if (blockInfo != null) { if (blockInfo == null)
return (BlockData) blockInfo.getA(); return null;
}
return null; return blockInfo.getBlockData();
} }
@Override @Override
@ -79,9 +76,9 @@ public class HSQLDBBlockArchiveRepository implements BlockArchiveRepository {
int height = referenceBlock.getHeight(); int height = referenceBlock.getHeight();
if (height > 0) { if (height > 0) {
// Request the block at height + 1 // Request the block at height + 1
Triple blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1); BlockTransformation blockInfo = BlockArchiveReader.getInstance().fetchBlockAtHeight(height + 1);
if (blockInfo != null) { if (blockInfo != null) {
return (BlockData) blockInfo.getA(); return blockInfo.getBlockData();
} }
} }
} }

View File

@ -0,0 +1,44 @@
package org.qortal.transform.block;
import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData;
import java.util.List;
public class BlockTransformation {
private final BlockData blockData;
private final List<TransactionData> transactions;
private final List<ATStateData> atStates;
private final byte[] atStatesHash;
/*package*/ BlockTransformation(BlockData blockData, List<TransactionData> transactions, List<ATStateData> atStates) {
this.blockData = blockData;
this.transactions = transactions;
this.atStates = atStates;
this.atStatesHash = null;
}
/*package*/ BlockTransformation(BlockData blockData, List<TransactionData> transactions, byte[] atStatesHash) {
this.blockData = blockData;
this.transactions = transactions;
this.atStates = null;
this.atStatesHash = atStatesHash;
}
public BlockData getBlockData() {
return blockData;
}
public List<TransactionData> getTransactions() {
return transactions;
}
public List<ATStateData> getAtStates() {
return atStates;
}
public byte[] getAtStatesHash() {
return atStatesHash;
}
}

View File

@ -3,12 +3,14 @@ package org.qortal.transform.block;
import java.io.ByteArrayOutputStream; import java.io.ByteArrayOutputStream;
import java.io.IOException; import java.io.IOException;
import java.nio.ByteBuffer; import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays; import java.util.Arrays;
import java.util.List; import java.util.List;
import org.qortal.block.Block; import org.qortal.block.Block;
import org.qortal.block.BlockChain; import org.qortal.block.BlockChain;
import org.qortal.crypto.Crypto;
import org.qortal.data.at.ATStateData; import org.qortal.data.at.ATStateData;
import org.qortal.data.block.BlockData; import org.qortal.data.block.BlockData;
import org.qortal.data.transaction.TransactionData; import org.qortal.data.transaction.TransactionData;
@ -20,7 +22,6 @@ import org.qortal.transform.Transformer;
import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
import org.qortal.utils.Serialization; import org.qortal.utils.Serialization;
import org.qortal.utils.Triple;
import com.google.common.primitives.Ints; import com.google.common.primitives.Ints;
import com.google.common.primitives.Longs; import com.google.common.primitives.Longs;
@ -45,14 +46,13 @@ public class BlockTransformer extends Transformer {
protected static final int AT_BYTES_LENGTH = INT_LENGTH; protected static final int AT_BYTES_LENGTH = INT_LENGTH;
protected static final int AT_FEES_LENGTH = AMOUNT_LENGTH; protected static final int AT_FEES_LENGTH = AMOUNT_LENGTH;
protected static final int AT_LENGTH = AT_FEES_LENGTH + AT_BYTES_LENGTH;
protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_COUNT_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_SIZE_LENGTH = INT_LENGTH;
protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = TIMESTAMP_LENGTH; protected static final int ONLINE_ACCOUNTS_TIMESTAMP_LENGTH = TIMESTAMP_LENGTH;
protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH; protected static final int ONLINE_ACCOUNTS_SIGNATURES_COUNT_LENGTH = INT_LENGTH;
protected static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH; public static final int AT_ENTRY_LENGTH = ADDRESS_LENGTH + SHA256_LENGTH + AMOUNT_LENGTH;
/** /**
* Extract block data and transaction data from serialized bytes. * Extract block data and transaction data from serialized bytes.
@ -61,7 +61,7 @@ public class BlockTransformer extends Transformer {
* @return BlockData and a List of transactions. * @return BlockData and a List of transactions.
* @throws TransformationException * @throws TransformationException
*/ */
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromBytes(byte[] bytes) throws TransformationException { public static BlockTransformation fromBytes(byte[] bytes) throws TransformationException {
if (bytes == null) if (bytes == null)
return null; return null;
@ -76,28 +76,40 @@ public class BlockTransformer extends Transformer {
/** /**
* Extract block data and transaction data from serialized bytes containing a single block. * Extract block data and transaction data from serialized bytes containing a single block.
* *
* @param bytes * @param byteBuffer source of serialized block bytes
* @return BlockData and a List of transactions. * @return BlockData and a List of transactions.
* @throws TransformationException * @throws TransformationException
*/ */
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException { public static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer) throws TransformationException {
return BlockTransformer.fromByteBuffer(byteBuffer, false);
}
/**
* Extract block data and transaction data from serialized bytes containing a single block.
*
* @param byteBuffer source of serialized block bytes
* @return BlockData and a List of transactions.
* @throws TransformationException
*/
public static BlockTransformation fromByteBufferV2(ByteBuffer byteBuffer) throws TransformationException {
return BlockTransformer.fromByteBuffer(byteBuffer, true); return BlockTransformer.fromByteBuffer(byteBuffer, true);
} }
/** /**
* Extract block data and transaction data from serialized bytes containing one or more blocks. * Extract block data and transaction data from serialized bytes containing a single block, in one of two forms.
* *
* @param bytes * @param byteBuffer source of serialized block bytes
* @param isV2 set to true if AT state info is represented by a single hash, false if serialized as per-AT address+state hash+fees
* @return the next block's BlockData and a List of transactions. * @return the next block's BlockData and a List of transactions.
* @throws TransformationException * @throws TransformationException
*/ */
public static Triple<BlockData, List<TransactionData>, List<ATStateData>> fromByteBuffer(ByteBuffer byteBuffer, boolean finalBlockInBuffer) throws TransformationException { private static BlockTransformation fromByteBuffer(ByteBuffer byteBuffer, boolean isV2) throws TransformationException {
int version = byteBuffer.getInt(); int version = byteBuffer.getInt();
if (finalBlockInBuffer && byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH) if (byteBuffer.remaining() < BASE_LENGTH + AT_BYTES_LENGTH - VERSION_LENGTH)
throw new TransformationException("Byte data too short for Block"); throw new TransformationException("Byte data too short for Block");
if (finalBlockInBuffer && byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize()) if (byteBuffer.remaining() > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for Block"); throw new TransformationException("Byte data too long for Block");
long timestamp = byteBuffer.getLong(); long timestamp = byteBuffer.getLong();
@ -117,42 +129,52 @@ public class BlockTransformer extends Transformer {
int atCount = 0; int atCount = 0;
long atFees = 0; long atFees = 0;
List<ATStateData> atStates = new ArrayList<>(); byte[] atStatesHash = null;
List<ATStateData> atStates = null;
int atBytesLength = byteBuffer.getInt(); if (isV2) {
// Simply: AT count, AT total fees, hash(all AT states)
atCount = byteBuffer.getInt();
atFees = byteBuffer.getLong();
atStatesHash = new byte[Transformer.SHA256_LENGTH];
byteBuffer.get(atStatesHash);
} else {
// V1: AT info byte length, then per-AT entries of AT address + state hash + fees
int atBytesLength = byteBuffer.getInt();
if (atBytesLength > BlockChain.getInstance().getMaxBlockSize())
throw new TransformationException("Byte data too long for Block's AT info");
if (atBytesLength > BlockChain.getInstance().getMaxBlockSize()) // Read AT-address, SHA256 hash and fees
throw new TransformationException("Byte data too long for Block's AT info"); if (atBytesLength % AT_ENTRY_LENGTH != 0)
throw new TransformationException("AT byte data not a multiple of AT entry length");
ByteBuffer atByteBuffer = byteBuffer.slice(); ByteBuffer atByteBuffer = byteBuffer.slice();
atByteBuffer.limit(atBytesLength); atByteBuffer.limit(atBytesLength);
// Read AT-address, SHA256 hash and fees atStates = new ArrayList<>();
if (atBytesLength % AT_ENTRY_LENGTH != 0) while (atByteBuffer.hasRemaining()) {
throw new TransformationException("AT byte data not a multiple of AT entry length"); byte[] atAddressBytes = new byte[ADDRESS_LENGTH];
atByteBuffer.get(atAddressBytes);
String atAddress = Base58.encode(atAddressBytes);
while (atByteBuffer.hasRemaining()) { byte[] stateHash = new byte[SHA256_LENGTH];
byte[] atAddressBytes = new byte[ADDRESS_LENGTH]; atByteBuffer.get(stateHash);
atByteBuffer.get(atAddressBytes);
String atAddress = Base58.encode(atAddressBytes);
byte[] stateHash = new byte[SHA256_LENGTH]; long fees = atByteBuffer.getLong();
atByteBuffer.get(stateHash);
long fees = atByteBuffer.getLong(); // Add this AT's fees to our total
atFees += fees;
// Add this AT's fees to our total atStates.add(new ATStateData(atAddress, stateHash, fees));
atFees += fees; }
atStates.add(new ATStateData(atAddress, stateHash, fees)); // Bump byteBuffer over AT states just read in slice
byteBuffer.position(byteBuffer.position() + atBytesLength);
// AT count to reflect the number of states we have
atCount = atStates.size();
} }
// Bump byteBuffer over AT states just read in slice
byteBuffer.position(byteBuffer.position() + atBytesLength);
// AT count to reflect the number of states we have
atCount = atStates.size();
// Add AT fees to totalFees // Add AT fees to totalFees
totalFees += atFees; totalFees += atFees;
@ -221,16 +243,15 @@ public class BlockTransformer extends Transformer {
byteBuffer.get(onlineAccountsSignatures); byteBuffer.get(onlineAccountsSignatures);
} }
// We should only complain about excess byte data if we aren't expecting more blocks in this ByteBuffer
if (finalBlockInBuffer && byteBuffer.hasRemaining())
throw new TransformationException("Excess byte data found after parsing Block");
// We don't have a height! // We don't have a height!
Integer height = null; Integer height = null;
BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp, BlockData blockData = new BlockData(version, reference, transactionCount, totalFees, transactionsSignature, height, timestamp,
minterPublicKey, minterSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures); minterPublicKey, minterSignature, atCount, atFees, encodedOnlineAccounts, onlineAccountsCount, onlineAccountsTimestamp, onlineAccountsSignatures);
return new Triple<>(blockData, transactions, atStates); if (isV2)
return new BlockTransformation(blockData, transactions, atStatesHash);
else
return new BlockTransformation(blockData, transactions, atStates);
} }
public static int getDataLength(Block block) throws TransformationException { public static int getDataLength(Block block) throws TransformationException {
@ -266,6 +287,14 @@ public class BlockTransformer extends Transformer {
} }
public static byte[] toBytes(Block block) throws TransformationException { public static byte[] toBytes(Block block) throws TransformationException {
return toBytes(block, false);
}
public static byte[] toBytesV2(Block block) throws TransformationException {
return toBytes(block, true);
}
private static byte[] toBytes(Block block, boolean isV2) throws TransformationException {
BlockData blockData = block.getBlockData(); BlockData blockData = block.getBlockData();
try { try {
@ -279,16 +308,37 @@ public class BlockTransformer extends Transformer {
bytes.write(blockData.getMinterSignature()); bytes.write(blockData.getMinterSignature());
int atBytesLength = blockData.getATCount() * AT_ENTRY_LENGTH; int atBytesLength = blockData.getATCount() * AT_ENTRY_LENGTH;
bytes.write(Ints.toByteArray(atBytesLength)); if (isV2) {
ByteArrayOutputStream atHashBytes = new ByteArrayOutputStream(atBytesLength);
long atFees = 0;
for (ATStateData atStateData : block.getATStates()) { for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block // Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial()) if (atStateData.isInitial())
continue; continue;
bytes.write(Base58.decode(atStateData.getATAddress())); atHashBytes.write(atStateData.getATAddress().getBytes(StandardCharsets.UTF_8));
bytes.write(atStateData.getStateHash()); atHashBytes.write(atStateData.getStateHash());
bytes.write(Longs.toByteArray(atStateData.getFees())); atHashBytes.write(Longs.toByteArray(atStateData.getFees()));
atFees += atStateData.getFees();
}
bytes.write(Ints.toByteArray(blockData.getATCount()));
bytes.write(Longs.toByteArray(atFees));
bytes.write(Crypto.digest(atHashBytes.toByteArray()));
} else {
bytes.write(Ints.toByteArray(atBytesLength));
for (ATStateData atStateData : block.getATStates()) {
// Skip initial states generated by DEPLOY_AT transactions in the same block
if (atStateData.isInitial())
continue;
bytes.write(Base58.decode(atStateData.getATAddress()));
bytes.write(atStateData.getStateHash());
bytes.write(Longs.toByteArray(atStateData.getFees()));
}
} }
// Transactions // Transactions

View File

@ -6,6 +6,7 @@ import org.qortal.data.transaction.TransactionData;
import org.qortal.repository.BlockArchiveReader; import org.qortal.repository.BlockArchiveReader;
import org.qortal.repository.DataException; import org.qortal.repository.DataException;
import org.qortal.repository.Repository; import org.qortal.repository.Repository;
import org.qortal.transform.block.BlockTransformation;
import java.util.List; import java.util.List;
@ -33,8 +34,7 @@ public class BlockArchiveUtils {
repository.discardChanges(); repository.discardChanges();
final int requestedRange = endHeight+1-startHeight; final int requestedRange = endHeight+1-startHeight;
List<Triple<BlockData, List<TransactionData>, List<ATStateData>>> blockInfoList = List<BlockTransformation> blockInfoList = BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight);
BlockArchiveReader.getInstance().fetchBlocksFromRange(startHeight, endHeight);
// Ensure that we have received all of the requested blocks // Ensure that we have received all of the requested blocks
if (blockInfoList == null || blockInfoList.isEmpty()) { if (blockInfoList == null || blockInfoList.isEmpty()) {
@ -43,27 +43,26 @@ public class BlockArchiveUtils {
if (blockInfoList.size() != requestedRange) { if (blockInfoList.size() != requestedRange) {
throw new IllegalStateException("Non matching block count when importing from archive"); throw new IllegalStateException("Non matching block count when importing from archive");
} }
Triple<BlockData, List<TransactionData>, List<ATStateData>> firstBlock = blockInfoList.get(0); BlockTransformation firstBlock = blockInfoList.get(0);
if (firstBlock == null || firstBlock.getA().getHeight() != startHeight) { if (firstBlock == null || firstBlock.getBlockData().getHeight() != startHeight) {
throw new IllegalStateException("Non matching first block when importing from archive"); throw new IllegalStateException("Non matching first block when importing from archive");
} }
if (blockInfoList.size() > 0) { if (blockInfoList.size() > 0) {
Triple<BlockData, List<TransactionData>, List<ATStateData>> lastBlock = BlockTransformation lastBlock = blockInfoList.get(blockInfoList.size() - 1);
blockInfoList.get(blockInfoList.size() - 1); if (lastBlock == null || lastBlock.getBlockData().getHeight() != endHeight) {
if (lastBlock == null || lastBlock.getA().getHeight() != endHeight) {
throw new IllegalStateException("Non matching last block when importing from archive"); throw new IllegalStateException("Non matching last block when importing from archive");
} }
} }
// Everything seems okay, so go ahead with the import // Everything seems okay, so go ahead with the import
for (Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo : blockInfoList) { for (BlockTransformation blockInfo : blockInfoList) {
try { try {
// Save block // Save block
repository.getBlockRepository().save(blockInfo.getA()); repository.getBlockRepository().save(blockInfo.getBlockData());
// Save AT state data hashes // Save AT state data hashes
for (ATStateData atStateData : blockInfo.getC()) { for (ATStateData atStateData : blockInfo.getAtStates()) {
atStateData.setHeight(blockInfo.getA().getHeight()); atStateData.setHeight(blockInfo.getBlockData().getHeight());
repository.getATRepository().save(atStateData); repository.getATRepository().save(atStateData);
} }

View File

@ -20,6 +20,7 @@ import org.qortal.test.common.Common;
import org.qortal.transaction.DeployAtTransaction; import org.qortal.transaction.DeployAtTransaction;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
import org.qortal.transform.TransformationException; import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.utils.BlockArchiveUtils; import org.qortal.utils.BlockArchiveUtils;
import org.qortal.utils.NTP; import org.qortal.utils.NTP;
import org.qortal.utils.Triple; import org.qortal.utils.Triple;
@ -123,8 +124,8 @@ public class BlockArchiveTests extends Common {
// Read block 2 from the archive // Read block 2 from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance(); BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> block2Info = reader.fetchBlockAtHeight(2); BlockTransformation block2Info = reader.fetchBlockAtHeight(2);
BlockData block2ArchiveData = block2Info.getA(); BlockData block2ArchiveData = block2Info.getBlockData();
// Read block 2 from the repository // Read block 2 from the repository
BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2); BlockData block2RepositoryData = repository.getBlockRepository().fromHeight(2);
@ -137,8 +138,8 @@ public class BlockArchiveTests extends Common {
assertEquals(1, block2ArchiveData.getOnlineAccountsCount()); assertEquals(1, block2ArchiveData.getOnlineAccountsCount());
// Read block 900 from the archive // Read block 900 from the archive
Triple<BlockData, List<TransactionData>, List<ATStateData>> block900Info = reader.fetchBlockAtHeight(900); BlockTransformation block900Info = reader.fetchBlockAtHeight(900);
BlockData block900ArchiveData = block900Info.getA(); BlockData block900ArchiveData = block900Info.getBlockData();
// Read block 900 from the repository // Read block 900 from the repository
BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900); BlockData block900RepositoryData = repository.getBlockRepository().fromHeight(900);
@ -200,10 +201,10 @@ public class BlockArchiveTests extends Common {
// Read a block from the archive // Read a block from the archive
BlockArchiveReader reader = BlockArchiveReader.getInstance(); BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(testHeight); BlockTransformation blockInfo = reader.fetchBlockAtHeight(testHeight);
BlockData archivedBlockData = blockInfo.getA(); BlockData archivedBlockData = blockInfo.getBlockData();
ATStateData archivedAtStateData = blockInfo.getC().isEmpty() ? null : blockInfo.getC().get(0); ATStateData archivedAtStateData = blockInfo.getAtStates().isEmpty() ? null : blockInfo.getAtStates().get(0);
List<TransactionData> archivedTransactions = blockInfo.getB(); List<TransactionData> archivedTransactions = blockInfo.getTransactions();
// Read the same block from the repository // Read the same block from the repository
BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight); BlockData repositoryBlockData = repository.getBlockRepository().fromHeight(testHeight);
@ -255,7 +256,7 @@ public class BlockArchiveTests extends Common {
// Check block 10 (unarchived) // Check block 10 (unarchived)
BlockArchiveReader reader = BlockArchiveReader.getInstance(); BlockArchiveReader reader = BlockArchiveReader.getInstance();
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = reader.fetchBlockAtHeight(10); BlockTransformation blockInfo = reader.fetchBlockAtHeight(10);
assertNull(blockInfo); assertNull(blockInfo);
} }

View File

@ -24,6 +24,7 @@ import org.qortal.test.common.TransactionUtils;
import org.qortal.transaction.Transaction; import org.qortal.transaction.Transaction;
import org.qortal.transaction.Transaction.TransactionType; import org.qortal.transaction.Transaction.TransactionType;
import org.qortal.transform.TransformationException; import org.qortal.transform.TransformationException;
import org.qortal.transform.block.BlockTransformation;
import org.qortal.transform.block.BlockTransformer; import org.qortal.transform.block.BlockTransformer;
import org.qortal.transform.transaction.TransactionTransformer; import org.qortal.transform.transaction.TransactionTransformer;
import org.qortal.utils.Base58; import org.qortal.utils.Base58;
@ -121,10 +122,10 @@ public class BlockTests extends Common {
assertEquals(BlockTransformer.getDataLength(block), bytes.length); assertEquals(BlockTransformer.getDataLength(block), bytes.length);
Triple<BlockData, List<TransactionData>, List<ATStateData>> blockInfo = BlockTransformer.fromBytes(bytes); BlockTransformation blockInfo = BlockTransformer.fromBytes(bytes);
// Compare transactions // Compare transactions
List<TransactionData> deserializedTransactions = blockInfo.getB(); List<TransactionData> deserializedTransactions = blockInfo.getTransactions();
assertEquals("Transaction count differs", blockData.getTransactionCount(), deserializedTransactions.size()); assertEquals("Transaction count differs", blockData.getTransactionCount(), deserializedTransactions.size());
for (int i = 0; i < blockData.getTransactionCount(); ++i) { for (int i = 0; i < blockData.getTransactionCount(); ++i) {