From 27b6b5ab97800e53b95535b6e222a509dcc24c62 Mon Sep 17 00:00:00 2001 From: Mike Hearn Date: Fri, 14 Oct 2011 12:33:47 +0000 Subject: [PATCH] Patch 8 from Steves lazy parsing patchset. More optimizations: pre-calculate or guess various array sizes to avoid needlessly re-sizing them later. Sha256Hash caches the hashCode. Message classes now track their (estimated) length even when not using deserialization-related constructors. --- .../google/bitcoin/core/AddressMessage.java | 18 +++++++++++ src/com/google/bitcoin/core/Block.java | 27 ++++++++++++++-- src/com/google/bitcoin/core/ChildMessage.java | 7 +++- src/com/google/bitcoin/core/EmptyMessage.java | 3 ++ .../google/bitcoin/core/GetBlocksMessage.java | 22 +++++++++++-- src/com/google/bitcoin/core/ListMessage.java | 5 +++ src/com/google/bitcoin/core/Message.java | 32 +++++++++++++------ src/com/google/bitcoin/core/PeerAddress.java | 4 ++- src/com/google/bitcoin/core/Sha256Hash.java | 13 ++++++-- src/com/google/bitcoin/core/Transaction.java | 7 ++-- .../google/bitcoin/core/TransactionInput.java | 7 ++++ .../bitcoin/core/TransactionOutPoint.java | 7 +++- .../bitcoin/core/TransactionOutput.java | 1 + src/com/google/bitcoin/core/VarInt.java | 14 +++++++- .../google/bitcoin/core/VersionMessage.java | 12 +++++++ 15 files changed, 156 insertions(+), 23 deletions(-) diff --git a/src/com/google/bitcoin/core/AddressMessage.java b/src/com/google/bitcoin/core/AddressMessage.java index 713ae29f..4e333fbd 100644 --- a/src/com/google/bitcoin/core/AddressMessage.java +++ b/src/com/google/bitcoin/core/AddressMessage.java @@ -48,6 +48,7 @@ public class AddressMessage extends Message { addresses.add(addr); cursor += addr.getMessageSize(); } + length = cursor - offset; } /* (non-Javadoc) @@ -63,6 +64,15 @@ public class AddressMessage extends Message { } } + + int getMessageSize() { + if (length != UNKNOWN_LENGTH) + return length; + length = new VarInt(addresses.size()).getSizeInBytes(); + if (addresses != null) + length += addresses.size() * (protocolVersion > 31402 ? PeerAddress.MESSAGE_SIZE : PeerAddress.MESSAGE_SIZE - 4); + return length; + } /** * @return An unmodifiableList view of the backing List of addresses. Addresses contained within the list may be safely modified. @@ -77,6 +87,10 @@ public class AddressMessage extends Message { checkParse(); address.setParent(this); addresses.add(address); + if (length == UNKNOWN_LENGTH) + getMessageSize(); + else + length += address.getMessageSize();; } public void removeAddress(int index) { @@ -84,6 +98,10 @@ public class AddressMessage extends Message { PeerAddress address = addresses.remove(index); if (address != null) address.setParent(null); + if (length == UNKNOWN_LENGTH) + getMessageSize(); + else + length -= address.getMessageSize(); } @Override diff --git a/src/com/google/bitcoin/core/Block.java b/src/com/google/bitcoin/core/Block.java index f82eefe8..0c04ced3 100644 --- a/src/com/google/bitcoin/core/Block.java +++ b/src/com/google/bitcoin/core/Block.java @@ -94,6 +94,8 @@ public class Block extends Message { difficultyTarget = 0x1d07fff8L; time = System.currentTimeMillis() / 1000; prevBlockHash = Sha256Hash.ZERO_HASH; + + length = 80; } /** Constructs a block object from the BitCoin wire format. */ @@ -364,8 +366,8 @@ public class Block extends Message { } // At least one of the two cacheable components is invalid - // so fall back to stream write since we can't be sure of the length. - ByteArrayOutputStream stream = new ByteArrayOutputStream(); + // so fall back to stream write since we can't be sure of the length. + ByteArrayOutputStream stream = new ByteArrayOutputStream(length == UNKNOWN_LENGTH ? 80 + guessTransactionsLength() : length); try { writeHeader(stream); writeTransactions(stream); @@ -381,6 +383,26 @@ public class Block extends Message { // We may only have enough data to write the header. writeTransactions(stream); } + + /** + * Provides a reasonable guess at the byte length of the transactions part of the block. + * The returned value will be accurate in 99% of cases and in those cases where not will probably + * slightly oversize. + * + * This is used to preallocate the underlying byte array for a ByteArrayOutputStream. If the size + * is under the real value the only penalty is resizing of the underlying byte array. + */ + private int guessTransactionsLength() { + if (transactionBytesValid) + return bytes.length - 80; + if (transactions == null) + return 0; + int len = VarInt.sizeOf(transactions.size()); + for (Transaction tx: transactions) { + len += tx.length == UNKNOWN_LENGTH ? 255 : tx.length; + } + return len; + } protected void unCache() { // Since we have alternate uncache methods to use internally this will @@ -772,6 +794,7 @@ public class Block extends Message { } t.setParent(this); transactions.add(t); + adjustLength(t.length); // Force a recalculation next time the values are needed. merkleRoot = null; hash = null; diff --git a/src/com/google/bitcoin/core/ChildMessage.java b/src/com/google/bitcoin/core/ChildMessage.java index 523035fa..5a9756df 100644 --- a/src/com/google/bitcoin/core/ChildMessage.java +++ b/src/com/google/bitcoin/core/ChildMessage.java @@ -58,6 +58,11 @@ public abstract class ChildMessage extends Message { parent.unCache(); } - + protected void adjustLength(int adjustment) { + if (length != UNKNOWN_LENGTH) + length += adjustment; + if (parent != null) + parent.adjustLength(adjustment); + } } diff --git a/src/com/google/bitcoin/core/EmptyMessage.java b/src/com/google/bitcoin/core/EmptyMessage.java index 69e3793b..50095f82 100644 --- a/src/com/google/bitcoin/core/EmptyMessage.java +++ b/src/com/google/bitcoin/core/EmptyMessage.java @@ -12,14 +12,17 @@ import java.io.OutputStream; public abstract class EmptyMessage extends Message { public EmptyMessage() { + length = 0; } public EmptyMessage(NetworkParameters params) { super(params); + length = 0; } public EmptyMessage(NetworkParameters params, byte[] msg, int offset) throws ProtocolException { super(params, msg, offset); + length = 0; } @Override diff --git a/src/com/google/bitcoin/core/GetBlocksMessage.java b/src/com/google/bitcoin/core/GetBlocksMessage.java index 13eb73a5..aa637806 100644 --- a/src/com/google/bitcoin/core/GetBlocksMessage.java +++ b/src/com/google/bitcoin/core/GetBlocksMessage.java @@ -18,20 +18,36 @@ package com.google.bitcoin.core; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; import java.util.List; public class GetBlocksMessage extends Message { private static final long serialVersionUID = 3479412877853645644L; - private final List locator; - private final Sha256Hash stopHash; + private long version; + private List locator; + private Sha256Hash stopHash; public GetBlocksMessage(NetworkParameters params, List locator, Sha256Hash stopHash) { super(params); + this.version = protocolVersion; this.locator = locator; this.stopHash = stopHash; } - public void parse() { + protected void parseLite() throws ProtocolException { + //NOP. This is a root level message and should always be provided with a length. + } + + public void parse() throws ProtocolException { + cursor = offset; + version = readUint32(); + int startCount = (int) readVarInt(); + if (startCount > 500) + throw new ProtocolException("Number of locators cannot be > 500, received: " + startCount);locator = new ArrayList(startCount); + for (int i = 0; i < startCount; i++) { + locator.add(readHash()); + } + stopHash = readHash(); } public List getLocator() { diff --git a/src/com/google/bitcoin/core/ListMessage.java b/src/com/google/bitcoin/core/ListMessage.java index d96c69d2..3f5a0675 100644 --- a/src/com/google/bitcoin/core/ListMessage.java +++ b/src/com/google/bitcoin/core/ListMessage.java @@ -50,6 +50,7 @@ public abstract class ListMessage extends Message public ListMessage(NetworkParameters params) { super(params); items = new ArrayList(); + length = 1; //length of 0 varint; } public List getItems() @@ -61,13 +62,17 @@ public abstract class ListMessage extends Message public void addItem(InventoryItem item) { unCache(); + length -= VarInt.sizeOf(items.size()); items.add(item); + length += VarInt.sizeOf(items.size()) + 36; } public void removeItem(int index) { unCache(); + length -= VarInt.sizeOf(items.size()); items.remove(index); + length += VarInt.sizeOf(items.size()) - 36; } @Override diff --git a/src/com/google/bitcoin/core/Message.java b/src/com/google/bitcoin/core/Message.java index 282cedc1..9684756c 100644 --- a/src/com/google/bitcoin/core/Message.java +++ b/src/com/google/bitcoin/core/Message.java @@ -140,9 +140,10 @@ public abstract class Message implements Serializable { * @return * @throws ProtocolException */ - protected void parseLite() throws ProtocolException { - length = getMessageSize(); - } + protected abstract void parseLite() throws ProtocolException; +// { +// length = getMessageSize(); +// } /** * Ensure the object is parsed if needed. This should be called in every getter before returning a value. @@ -209,6 +210,12 @@ public abstract class Message implements Serializable { recached = false; } + protected void adjustLength(int adjustment) { + if (length != UNKNOWN_LENGTH) + //our own length is now unknown if we have an unknown length adjustment. + length = adjustment == UNKNOWN_LENGTH ? UNKNOWN_LENGTH : length + adjustment; + } + /** * used for unit testing */ @@ -263,7 +270,7 @@ public abstract class Message implements Serializable { assert bytes == null : "cached bytes present but failed to use them for serialization"; //no cached array available so serialize parts by stream. - ByteArrayOutputStream stream = new ByteArrayOutputStream(); + ByteArrayOutputStream stream = new ByteArrayOutputStream(length < 32 ? 32 : length + 32); try { bitcoinSerializeToStream(stream); } catch (IOException e) { @@ -286,8 +293,12 @@ public abstract class Message implements Serializable { length = bytes.length; return bytes; } - - return stream.toByteArray(); + //record length. If this Message wasn't parsed from a but stream it won't have length field + //set (except for static length message types). Setting it makes future streaming more efficient + //because we can preallocate the ByteArrayOutputStream buffer and avoid resizing. + byte[] buf = stream.toByteArray(); + length = buf.length; + return buf; } /** @@ -322,17 +333,18 @@ public abstract class Message implements Serializable { } /** - * This should be overidden to extract correct message size in the case of lazy parsing. Until this method is - * implemented in a subclass of ChildMessage lazy parsing will have no effect. + * This should be overridden to extract correct message size in the case of lazy parsing. Until this method is + * implemented in a subclass of ChildMessage lazy parsing may have no effect. * - * This default implementation is a safe fall back that will ensure it returns a correct value. + * This default implementation is a safe fall back that will ensure it returns a correct value by parsing the message. * @return */ int getMessageSize() { if (length != UNKNOWN_LENGTH) return length; checkParse(); - length = cursor - offset; + if (length != UNKNOWN_LENGTH) + length = cursor - offset; return length; } diff --git a/src/com/google/bitcoin/core/PeerAddress.java b/src/com/google/bitcoin/core/PeerAddress.java index c47c1758..c199ffbb 100644 --- a/src/com/google/bitcoin/core/PeerAddress.java +++ b/src/com/google/bitcoin/core/PeerAddress.java @@ -33,7 +33,7 @@ import static com.google.bitcoin.core.Utils.uint64ToByteStreamLE; */ public class PeerAddress extends ChildMessage { private static final long serialVersionUID = 7501293709324197411L; - private static final int MESSAGE_SIZE = 30; + static final int MESSAGE_SIZE = 30; private InetAddress addr; private int port; @@ -68,6 +68,7 @@ public class PeerAddress extends ChildMessage { this.port = port; this.protocolVersion = protocolVersion; this.services = BigInteger.ZERO; + length = protocolVersion > 31402 ? MESSAGE_SIZE : MESSAGE_SIZE - 4; } public PeerAddress(InetAddress addr, int port) { @@ -137,6 +138,7 @@ public class PeerAddress extends ChildMessage { */ @Override int getMessageSize() { + length = protocolVersion > 31402 ? MESSAGE_SIZE : MESSAGE_SIZE - 4; return length; } diff --git a/src/com/google/bitcoin/core/Sha256Hash.java b/src/com/google/bitcoin/core/Sha256Hash.java index 2a2eded5..821fc129 100644 --- a/src/com/google/bitcoin/core/Sha256Hash.java +++ b/src/com/google/bitcoin/core/Sha256Hash.java @@ -32,6 +32,7 @@ public class Sha256Hash implements Serializable { private static final long serialVersionUID = 3778897922647016546L; private byte[] bytes; + private int hash = -1; public static final Sha256Hash ZERO_HASH = new Sha256Hash(new byte[32]); @@ -40,6 +41,12 @@ public class Sha256Hash implements Serializable { assert bytes.length == 32; this.bytes = bytes; } + + private Sha256Hash(byte[] bytes, int hash) { + assert bytes.length == 32; + this.bytes = bytes; + this.hash = hash; + } /** Creates a Sha256Hash by decoding the given hex string. It must be 64 characters long. */ public Sha256Hash(String string) { @@ -71,7 +78,9 @@ public class Sha256Hash implements Serializable { */ @Override public int hashCode() { - return Arrays.hashCode(bytes); + if (hash == -1) + hash = Arrays.hashCode(bytes); + return hash; } @Override @@ -89,6 +98,6 @@ public class Sha256Hash implements Serializable { } public Sha256Hash duplicate() { - return new Sha256Hash(bytes); + return new Sha256Hash(bytes, hash); } } diff --git a/src/com/google/bitcoin/core/Transaction.java b/src/com/google/bitcoin/core/Transaction.java index 12213a76..37aa4d26 100644 --- a/src/com/google/bitcoin/core/Transaction.java +++ b/src/com/google/bitcoin/core/Transaction.java @@ -79,6 +79,7 @@ public class Transaction extends ChildMessage implements Serializable { inputs = new ArrayList(); outputs = new ArrayList(); // We don't initialize appearsIn deliberately as it's only useful for transactions stored in the wallet. + length = 10; //8 for std fields + 1 for each 0 varint } /** @@ -461,6 +462,7 @@ public class Transaction extends ChildMessage implements Serializable { input.setParent(this); immutableInputs = null; inputs.add(input); + adjustLength(input.length); } /** @@ -475,6 +477,7 @@ public class Transaction extends ChildMessage implements Serializable { immutableOutputs = null; outputs.add(to); + adjustLength(to.length); } /** @@ -549,8 +552,8 @@ public class Transaction extends ChildMessage implements Serializable { private byte[] hashTransactionForSignature(SigHash type, boolean anyoneCanPay) { try { - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - bitcoinSerializeToStream(bos); + ByteArrayOutputStream bos = new ByteArrayOutputStream(length == UNKNOWN_LENGTH ? 256 : length + 4); + bitcoinSerialize(bos); // We also have to write a hash type. int hashType = type.ordinal() + 1; if (anyoneCanPay) diff --git a/src/com/google/bitcoin/core/TransactionInput.java b/src/com/google/bitcoin/core/TransactionInput.java index 26966708..18867a20 100644 --- a/src/com/google/bitcoin/core/TransactionInput.java +++ b/src/com/google/bitcoin/core/TransactionInput.java @@ -55,6 +55,8 @@ public class TransactionInput extends ChildMessage implements Serializable { this.outpoint = new TransactionOutPoint(params, -1, null); this.sequence = 0xFFFFFFFFL; this.parentTransaction = parentTransaction; + + length = 40 + (scriptBytes == null ? 1 : VarInt.sizeOf(scriptBytes.length) + scriptBytes.length); } /** Creates an UNSIGNED input that links to the given output */ @@ -65,6 +67,8 @@ public class TransactionInput extends ChildMessage implements Serializable { scriptBytes = EMPTY_ARRAY; sequence = 0xFFFFFFFFL; this.parentTransaction = parentTransaction; + + length = 41; } /** Deserializes an input message. This is usually part of a transaction message. */ @@ -172,7 +176,10 @@ public class TransactionInput extends ChildMessage implements Serializable { */ void setScriptBytes(byte[] scriptBytes) { unCache(); + int oldLength = length; this.scriptBytes = scriptBytes; + int newLength = 40 + (scriptBytes == null ? 1 : VarInt.sizeOf(scriptBytes.length) + scriptBytes.length); + adjustLength(newLength - oldLength); } /** diff --git a/src/com/google/bitcoin/core/TransactionOutPoint.java b/src/com/google/bitcoin/core/TransactionOutPoint.java index a2a7dc46..c1c0f8a5 100644 --- a/src/com/google/bitcoin/core/TransactionOutPoint.java +++ b/src/com/google/bitcoin/core/TransactionOutPoint.java @@ -29,7 +29,7 @@ import java.io.Serializable; public class TransactionOutPoint extends ChildMessage implements Serializable { private static final long serialVersionUID = -6320880638344662579L; - private static final int MESSAGE_LENGTH = 36; + static final int MESSAGE_LENGTH = 36; /** Hash of the transaction to which we refer. */ private Sha256Hash hash; @@ -50,6 +50,7 @@ public class TransactionOutPoint extends ChildMessage implements Serializable { // This happens when constructing the genesis block. hash = Sha256Hash.ZERO_HASH; } + length = MESSAGE_LENGTH; } /** Deserializes the message. This is usually part of a transaction message. */ @@ -62,6 +63,10 @@ public class TransactionOutPoint extends ChildMessage implements Serializable { super(params, payload, offset, parent, parseLazy, parseRetain, MESSAGE_LENGTH); } + protected void parseLite() throws ProtocolException { + length = MESSAGE_LENGTH; + } + @Override void parse() throws ProtocolException { hash = readHash(); diff --git a/src/com/google/bitcoin/core/TransactionOutput.java b/src/com/google/bitcoin/core/TransactionOutput.java index fc2407e2..0362e18d 100644 --- a/src/com/google/bitcoin/core/TransactionOutput.java +++ b/src/com/google/bitcoin/core/TransactionOutput.java @@ -74,6 +74,7 @@ public class TransactionOutput extends ChildMessage implements Serializable { this.scriptBytes = Script.createOutputScript(to); parentTransaction = parent; availableForSpending = true; + length = 8 + VarInt.sizeOf(scriptBytes.length) + scriptBytes.length; } /** Used only in creation of the genesis blocks and in unit tests. */ diff --git a/src/com/google/bitcoin/core/VarInt.java b/src/com/google/bitcoin/core/VarInt.java index cc56e3ef..1be6e13e 100644 --- a/src/com/google/bitcoin/core/VarInt.java +++ b/src/com/google/bitcoin/core/VarInt.java @@ -46,6 +46,19 @@ public class VarInt { } public int getSizeInBytes() { + return sizeOf(value); + } + + public static int sizeOf(int value) { + // Java doesn't have the actual value of MAX_INT, as all types in Java are signed. + if (value < 253) + return 1; + else if (value < 65536) + return 3; // 1 marker + 2 data bytes + return 5; // 1 marker + 4 data bytes + } + + public static int sizeOf(long value) { // Java doesn't have the actual value of MAX_INT, as all types in Java are signed. if (isLessThanUnsigned(value, 253)) return 1; @@ -56,7 +69,6 @@ public class VarInt { else return 9; // 1 marker + 8 data bytes } - public byte[] encode() { return encodeBE(); diff --git a/src/com/google/bitcoin/core/VersionMessage.java b/src/com/google/bitcoin/core/VersionMessage.java index 30ecb70f..d250de11 100644 --- a/src/com/google/bitcoin/core/VersionMessage.java +++ b/src/com/google/bitcoin/core/VersionMessage.java @@ -75,7 +75,17 @@ public class VersionMessage extends Message { } subVer = "BitCoinJ 0.3-SNAPSHOT"; bestHeight = newBestHeight; + + length = 84; + if (protocolVersion > 31402) + length += 8; + length += subVer == null ? 1 : VarInt.sizeOf(subVer.length()) + subVer.length(); } + + @Override + protected void parseLite() throws ProtocolException { + //NOP. VersionMessage is never lazy parsed. + } @Override public void parse() throws ProtocolException { @@ -98,6 +108,7 @@ public class VersionMessage extends Message { subVer = readStr(); // int bestHeight (size of known block chain). bestHeight = readUint32(); + length = cursor - offset; } @Override @@ -169,4 +180,5 @@ public class VersionMessage extends Message { sb.append("best height: ").append(bestHeight).append("\n"); return sb.toString(); } + }