Qortal/altcoinj
3
0
Fork 0
mirror of https://github.com/Qortal/altcoinj.git synced 2025-02-07 06:44:16 +00:00
Code Issues Projects Releases Wiki Activity

Remove deduplication code. It is dead for a long time now already.

Browse Source
This commit is contained in:
Mike Hearn 2012-05-21 16:10:28 +02:00
parent e42063806c
commit e375270d6b
6 changed files with 40 additions and 136 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
core/src/main/java/com/google/bitcoin/core/BitcoinSerializer.java
View File

@ -26,7 +26,6 @@ import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
import static com.google.bitcoin.core.Utils.*;
@ -72,36 +71,14 @@ public class BitcoinSerializer {
names.put(HeadersMessage.class, "headers");
}
/**
* A doubly-linked map of message-hash to counts. When a new message is received we increment the count in
* this list. The count isn't currently used, but will be helpful later to know how many peers relayed a
* particular transaction. We can use that as a heuristic to estimate validity.
*/
private LinkedHashMap<Sha256Hash, Integer> dedupeList;
/*
* Returns a {@link LinkedHashMap} that evicts old entries, making it suitable for passing to the constructor
* if you wish to use message deduplication.
*/
public static LinkedHashMap<Sha256Hash, Integer> createDedupeList() {
return new LinkedHashMap<Sha256Hash, Integer>() {
@Override
protected boolean removeEldestEntry(Map.Entry<Sha256Hash, Integer> entry) {
// Keep 100 message hashcodes in the list. This choice is fairly arbitrary.
return size() > 100;
}
};
}
/**
* Constructs a BitcoinSerializer with the given behavior.
*
* @param params networkParams used to create Messages instances and termining packetMagic
* @param usesChecksumming set to true if checkums should be included and expected in headers
*/
public BitcoinSerializer(NetworkParameters params, boolean usesChecksumming,
LinkedHashMap<Sha256Hash, Integer> dedupeList) {
this(params, usesChecksumming, false, false, dedupeList);
public BitcoinSerializer(NetworkParameters params, boolean usesChecksumming) {
this(params, usesChecksumming, false, false);
}
/**
@ -111,13 +88,11 @@ public class BitcoinSerializer {
* @param usesChecksumming set to true if checkums should be included and expected in headers
* @param parseLazy deserialize messages in lazy mode.
* @param parseRetain retain the backing byte array of a message for fast reserialization.
* @param dedupeList possibly shared list of previously received messages used to avoid parsing duplicates.
*/
public BitcoinSerializer(NetworkParameters params, boolean usesChecksumming, boolean parseLazy, boolean parseRetain,
LinkedHashMap<Sha256Hash, Integer> dedupeList) {
public BitcoinSerializer(NetworkParameters params, boolean usesChecksumming,
boolean parseLazy, boolean parseRetain) {
this.params = params;
this.usesChecksumming = usesChecksumming;
this.dedupeList = dedupeList;
this.parseLazy = parseLazy;
this.parseRetain = parseRetain;
}
@ -197,11 +172,10 @@ public class BitcoinSerializer {
}
/**
* Reads a message from the given InputStream and returns it. If deduping is enabled and the message has already
* been parsed/returned, it will return null.
* Reads a message from the given InputStream and returns it.
*/
public Message deserialize(InputStream in) throws ProtocolException, IOException {
// A BitCoin protocol message has the following format.
// A Bitcoin protocol message has the following format.
//
// - 4 byte magic number: 0xfabfb5da for the testnet or
// 0xf9beb4d9 for production
@ -221,13 +195,6 @@ public class BitcoinSerializer {
return deserializePayload(header, in);
}
private boolean canDedupeMessageType(String command) {
// We don't attempt to deduplicate messages that may be legitimately duplicated like ping or versions nor do
// we dedupe addr messages which are always different even if they contain redundant data. Trying to dedupe
// them would just fill up the shared hashmap.
return command.equals("block") || command.equals("tx");
}
/**
* Deserializes only the header in case packet meta data is needed before decoding
* the payload. This method assumes you have already called seekPastMagicBytes()
@ -238,8 +205,7 @@ public class BitcoinSerializer {
/**
* Deserialize payload only. You must provide a header, typically obtained by calling
* {@link BitcoinSerializer#deserializeHeader}. If the deduping feature is active, may return NULL if the
* message was seen before.
* {@link BitcoinSerializer#deserializeHeader}.
*/
public Message deserializePayload(BitcoinPacketHeader header, InputStream in) throws ProtocolException, IOException {
int readCursor = 0;
@ -252,41 +218,10 @@ public class BitcoinSerializer {
readCursor += bytesRead;
}
// Check for duplicates. This is to avoid the cost (cpu and memory) of parsing the message twice, which can
// be an issue on constrained devices.
//save this for reuse later. Hashing is expensive so checksumming starting with a single hash
//is a significant saving.
Sha256Hash singleHash = null;
if (dedupeList != null && canDedupeMessageType(header.command)) {
// We use a secure hash here rather than the faster and simpler array hashes because otherwise a malicious
// node on the network could broadcast a message designed to mask a different message. They would not
// necessarily have to be connected directly to this program.
synchronized (dedupeList) {
// Calculate hash inside the lock to avoid unnecessary battery power spent on hashing messages arriving
// on different threads simultaneously.
singleHash = Sha256Hash.create(payloadBytes);
Integer count = dedupeList.get(singleHash);
if (count != null) {
int newCount = count + 1;
log.info("Received duplicate {} message, now seen {} times", header.command, newCount);
dedupeList.put(singleHash, newCount);
return null;
} else {
dedupeList.put(singleHash, 1);
}
}
}
// Verify the checksum.
byte[] hash = null;
if (usesChecksumming) {
if (singleHash != null) {
hash = singleDigest(singleHash.getBytes(), 0, 32);
} else {
hash = doubleDigest(payloadBytes);
}
hash = doubleDigest(payloadBytes);
if (header.checksum[0] != hash[0] || header.checksum[1] != hash[1] ||
header.checksum[2] != hash[2] || header.checksum[3] != hash[3]) {
throw new ProtocolException("Checksum failed to verify, actual " +

2
core/src/main/java/com/google/bitcoin/core/Peer.java
View File

@ -148,7 +148,7 @@ public class Peer {
*/
public synchronized void connect() throws PeerException {
try {
conn = new TCPNetworkConnection(params, false, versionMessage);
conn = new TCPNetworkConnection(params, versionMessage);
conn.connect(address, CONNECT_TIMEOUT_MSEC);
} catch (IOException ex) {
throw new PeerException(ex);

49
core/src/main/java/com/google/bitcoin/core/TCPNetworkConnection.java
View File

@ -26,7 +26,6 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.util.Date;
import java.util.LinkedHashMap;
/**
* A {@code TCPNetworkConnection} is used for connecting to a Bitcoin node over the standard TCP/IP protocol.<p>
@ -47,8 +46,6 @@ public class TCPNetworkConnection implements NetworkConnection {
private final NetworkParameters params;
private VersionMessage versionMessage;
// Given to the BitcoinSerializer to de-duplicate messages.
private static final LinkedHashMap<Sha256Hash, Integer> dedupeList = BitcoinSerializer.createDedupeList();
private BitcoinSerializer serializer = null;
private VersionMessage myVersionMessage;
@ -61,13 +58,11 @@ public class TCPNetworkConnection implements NetworkConnection {
* @param peerAddress address to connect to. IPv6 is not currently supported by BitCoin. If
* port is not positive the default port from params is used.
* @param params Defines which network to connect to and details of the protocol.
* @param connectTimeoutMsec Timeout in milliseconds when initially connecting to peer
* @param dedupe Whether to avoid parsing duplicate messages from the network (ie from other peers).
* @param ver The VersionMessage to announce to the other side of the connection.
* @throws IOException if there is a network related failure.
* @throws ProtocolException if the version negotiation failed.
*/
public TCPNetworkConnection(NetworkParameters params, boolean dedupe, VersionMessage ver)
public TCPNetworkConnection(NetworkParameters params, VersionMessage ver)
throws IOException, ProtocolException {
this.params = params;
this.myVersionMessage = ver;
@ -75,12 +70,25 @@ public class TCPNetworkConnection implements NetworkConnection {
socket = new Socket();
// So pre-Feb 2012, update checkumming property after version is read.
this.serializer = new BitcoinSerializer(this.params, false, dedupe ? dedupeList : null);
this.serializer = new BitcoinSerializer(this.params, false);
this.serializer.setUseChecksumming(Utils.now().after(checksummingProtocolChangeDate));
}
public void connect(PeerAddress peerAddress, int connectTimeoutMsec)
/**
* Connect to the given IP address using the port specified as part of the network parameters. Once construction
* is complete a functioning network channel is set up and running.
*
* @param params Defines which network to connect to and details of the protocol.
* @param bestHeight The height of the best chain we know about, sent to the other side.
* @throws IOException if there is a network related failure.
* @throws ProtocolException if the version negotiation failed.
*/
public TCPNetworkConnection(NetworkParameters params, int bestHeight)
throws IOException, ProtocolException {
this(params, new VersionMessage(params, bestHeight));
}
public void connect(PeerAddress peerAddress, int connectTimeoutMsec) throws IOException, ProtocolException {
remoteIp = peerAddress.getAddr();
int port = (peerAddress.getPort() > 0) ? peerAddress.getPort() : this.params.port;
@ -136,31 +144,6 @@ public class TCPNetworkConnection implements NetworkConnection {
// Handshake is done!
}
/**
* Connect to the given IP address using the port specified as part of the network parameters. Once construction
* is complete a functioning network channel is set up and running.
*
* @param peerAddress address to connect to. IPv6 is not currently supported by BitCoin. If
* port is not positive the default port from params is used.
* @param params Defines which network to connect to and details of the protocol.
* @param connectTimeoutMsec Timeout in milliseconds when initially connecting to peer
* @param dedupe Whether to avoid parsing duplicate messages from the network (ie from other peers).
* @param bestHeight The height of the best chain we know about, sent to the other side.
* @throws IOException if there is a network related failure.
* @throws ProtocolException if the version negotiation failed.
*/
public TCPNetworkConnection(NetworkParameters params,
int bestHeight, boolean dedupe)
throws IOException, ProtocolException {
this(params, dedupe, new VersionMessage(params, bestHeight));
}
public TCPNetworkConnection(NetworkParameters params, int bestHeight)
throws IOException, ProtocolException {
this(params, bestHeight, true);
}
public void ping() throws IOException {
writeMessage(new Ping());
}

32
core/src/test/java/com/google/bitcoin/core/BitcoinSerializerTest.java
View File

@ -23,9 +23,9 @@ import org.spongycastle.util.encoders.Hex;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.util.Arrays;
import java.util.LinkedHashMap;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
public class BitcoinSerializerTest {
private final byte[] addrMessage = Hex.decode("f9beb4d96164647200000000000000001f000000" +
@ -53,7 +53,7 @@ public class BitcoinSerializerTest {
@Test
public void testVersion() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), false, null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), false);
// the actual data from https://en.bitcoin.it/wiki/Protocol_specification#version
ByteArrayInputStream bais = new ByteArrayInputStream(Hex.decode("f9beb4d976657273696f6e0000000000550000009" +
"c7c00000100000000000000e615104d00000000010000000000000000000000000000000000ffff0a000001daf6010000" +
@ -73,7 +73,7 @@ public class BitcoinSerializerTest {
@Test
public void testVerack() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), false, null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), false);
// the actual data from https://en.bitcoin.it/wiki/Protocol_specification#verack
ByteArrayInputStream bais = new ByteArrayInputStream(Hex.decode("f9beb4d976657261636b00000000000000000000"));
VersionAck va = (VersionAck)bs.deserialize(bais);
@ -82,7 +82,7 @@ public class BitcoinSerializerTest {
@Test
public void testAddr() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true);
// the actual data from https://en.bitcoin.it/wiki/Protocol_specification#addr
ByteArrayInputStream bais = new ByteArrayInputStream(addrMessage);
AddressMessage a = (AddressMessage)bs.deserialize(bais);
@ -98,20 +98,8 @@ public class BitcoinSerializerTest {
}
@Test
public void testDeduplication() throws Exception {
LinkedHashMap<Sha256Hash, Integer> dedupeList = BitcoinSerializer.createDedupeList();
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, dedupeList);
ByteArrayInputStream bais = new ByteArrayInputStream(txMessage);
Transaction tx = (Transaction)bs.deserialize(bais);
assertNotNull(tx);
bais.reset();
tx = (Transaction)bs.deserialize(bais);
assertNull(tx);
}
@Test
public void testLazyParsing() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, true, false, null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, true, false);
ByteArrayInputStream bais = new ByteArrayInputStream(txMessage);
Transaction tx = (Transaction)bs.deserialize(bais);
@ -136,7 +124,7 @@ public class BitcoinSerializerTest {
}
private void testCachedParsing(boolean lazy) throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, lazy, true, null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true, lazy, true);
//first try writing to a fields to ensure uncaching and children are not affected
ByteArrayInputStream bais = new ByteArrayInputStream(txMessage);
@ -203,8 +191,7 @@ public class BitcoinSerializerTest {
*/
@Test
public void testHeaders1() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true,
null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true);
ByteArrayInputStream bais = new ByteArrayInputStream(Hex.decode("f9beb4d9686561" +
"646572730000000000520000005d4fab8101010000006fe28c0ab6f1b372c1a6a246ae6" +
@ -231,8 +218,7 @@ public class BitcoinSerializerTest {
* Get 6 headers of blocks 1-6 in the chain
*/
public void testHeaders2() throws Exception {
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true,
null);
BitcoinSerializer bs = new BitcoinSerializer(NetworkParameters.prodNet(), true);
ByteArrayInputStream bais = new ByteArrayInputStream(Hex.decode("f9beb4d96865616465" +
"72730000000000e701000085acd4ea06010000006fe28c0ab6f1b372c1a6a246ae63f74f931e" +

10
core/src/test/java/com/google/bitcoin/core/LazyParseByteCacheTest.java
View File

@ -99,7 +99,7 @@ public class LazyParseByteCacheTest {
Block b1 = createFakeBlock(unitTestParams, blockStore, tx1, tx2).block;
BitcoinSerializer bs = new BitcoinSerializer(unitTestParams, true, null);
BitcoinSerializer bs = new BitcoinSerializer(unitTestParams, true);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
bs.serialize(tx1, bos);
@ -171,10 +171,10 @@ public class LazyParseByteCacheTest {
public void testBlock(byte[] blockBytes, boolean isChild, boolean lazy, boolean retain) throws Exception {
//reference serializer to produce comparison serialization output after changes to
//message structure.
BitcoinSerializer bsRef = new BitcoinSerializer(unitTestParams, true, false, false, null);
BitcoinSerializer bsRef = new BitcoinSerializer(unitTestParams, true, false, false);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
BitcoinSerializer bs = new BitcoinSerializer(unitTestParams, true, lazy, retain, null);
BitcoinSerializer bs = new BitcoinSerializer(unitTestParams, true, lazy, retain);
Block b1;
Block bRef;
b1 = (Block) bs.deserialize(new ByteArrayInputStream(blockBytes));
@ -407,10 +407,10 @@ public class LazyParseByteCacheTest {
//reference serializer to produce comparison serialization output after changes to
//message structure.
BitcoinSerializer bsRef = new BitcoinSerializer(params, true, false, false, null);
BitcoinSerializer bsRef = new BitcoinSerializer(params, true, false, false);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
BitcoinSerializer bs = new BitcoinSerializer(params, true, lazy, retain, null);
BitcoinSerializer bs = new BitcoinSerializer(params, true, lazy, retain);
Transaction t1;
Transaction tRef;
t1 = (Transaction) bs.deserialize(new ByteArrayInputStream(txBytes));

2
core/src/test/java/com/google/bitcoin/core/TestUtils.java
View File

@ -80,7 +80,7 @@ public class TestUtils {
* Roundtrip a transaction so that it appears as if it has just come from the wire
*/
private static Transaction roundTripTransaction(NetworkParameters params, Transaction tx) throws IOException, ProtocolException {
BitcoinSerializer bs = new BitcoinSerializer(params, true, null);
BitcoinSerializer bs = new BitcoinSerializer(params, true);
ByteArrayOutputStream bos = new ByteArrayOutputStream();
bs.serialize(tx, bos);