Add BloomFilter class/messages and a test-case.

core/src/main/java/com/google/bitcoin/core/BitcoinSerializer.java

@@ -70,6 +70,7 @@ public class BitcoinSerializer {
         names.put(GetHeadersMessage.class, "getheaders");
         names.put(GetAddrMessage.class, "getaddr");
         names.put(HeadersMessage.class, "headers");
+        names.put(BloomFilter.class, "filterload");
     }
 
     /**
@@ -274,6 +275,8 @@ public class BitcoinSerializer {
             return new HeadersMessage(params, payloadBytes);
         } else if (command.equals("alert")) {
             return new AlertMessage(params, payloadBytes);
+        } else if (command.equals("filterload")) {
+            return new BloomFilter(params, payloadBytes);
         } else {
             log.warn("No support for deserializing message with name {}", command);
             return new UnknownMessage(params, command, payloadBytes);

core/src/main/java/com/google/bitcoin/core/BloomFilter.java

@@ -0,0 +1,222 @@
+/*
+ * Copyright 2012 Matt Corallo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.bitcoin.core;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * <p>BloomFilter is a probabilistic filter which can be sent to another client
+ * so that it can filter the transactions it sends us.</p>
+ * 
+ * <p>This allows for significantly more efficient transaction and block downloads.</p>
+ * 
+ * <p>By inserting the full set of hashed and full public keys in all of our wallets,
+ * a node which has a copy of a BloomFilter will filter transaction inv messages
+ * to only those which are to or from one of our wallets.</p>
+ * 
+ * <p>In order to retain anonymity, the false positive rate can be increased.
+ * This will decrease the download efficiency by increasing the number of transactions
+ * which match the filter but are not actually ours.</p>
+ */
+public class BloomFilter extends Message {
+    private byte[] data;
+    private long hashFuncs;
+    private long nTweak;
+
+    // Same value as the reference client
+    // A filter of 20,000 items and a false positive rate of 0.1% or one of 10,000 items and 0.0001% is just under 36,000 bytes
+    private static final long MAX_FILTER_SIZE = 36000;
+    // There is little reason to ever have more hash functions than 50 given a limit of 36,000 bytes
+    private static final int MAX_HASH_FUNCS = 50;
+
+    /**
+     * Construct a BloomFilter by deserializing payloadBytes
+     */
+    public BloomFilter(NetworkParameters params, byte[] payloadBytes) throws ProtocolException {
+        super(params, payloadBytes, 0);
+    }
+    
+    /**
+     * <p>Constructs a new Bloom Filter which will provide approximately the given false positive
+     * rate when the given number of elements have been inserted.</p>
+     * 
+     * <p>If the filter would otherwise be larger than the maximum allowed size, it will be
+     * automatically downsized to the maximum size.</p>
+     * 
+     * <p>To check the theoretical false positive rate of a given filter, use {@link BloomFilter#getFalsePositiveRate(int)}</p>
+     * 
+     * <p>The anonymity of which coins are yours to any peer which you send a BloomFilter to is
+     * controlled by the false positive rate.</p>
+     * 
+     * <p>For reference, as of block 187,000, the total number of addresses used in the chain was roughly 4.5 million.</p>
+     * 
+     * <p>Thus, if you use a false positive rate of 0.001 (0.1%), there will be, on average, 4,500 distinct public
+     * keys/addresses which will be thought to be yours by nodes which have your bloom filter, but which are not
+     * actually yours.</p>
+     * 
+     * <p>Keep in mind that a remote node can do a pretty good job estimating the order of magnitude of the false positive
+     * rate of a given filter you provide it when considering the anonymity of a given filter.</p>
+     * 
+     * <p>randomNonce is a tweak for the hash function used to prevent some theoretical DoS attacks.
+     * It should be a random value, however secureness of the random value is of no great consequence.</p>
+     */
+    public BloomFilter(int elements, double falsePositiveRate, long randomNonce) {
+        // The following formulas were stolen from Wikipedia's page on Bloom Filters (with the addition of min(..., MAX_...))
+        //                        Size required for a given number of elements and false-positive rate
+        int size = Math.min((int)(-1  / (Math.pow(Math.log(2), 2)) * elements * Math.log(falsePositiveRate)),
+                            (int)MAX_FILTER_SIZE * 8) / 8;
+        data = new byte[size <= 0 ? 1 : size];
+        // Optimal number of hash functions for a given filter size and element count.
+        hashFuncs = Math.min((int)(data.length * 8 / elements * Math.log(2)), MAX_HASH_FUNCS);
+        this.nTweak = randomNonce;
+    }
+    
+    /**
+     * Returns the theoretical false positive rate of this filter if were to contain the given number of elements.
+     */
+    public double getFalsePositiveRate(int elements) {
+        return Math.pow(1 - Math.pow(Math.E, -1.0 * (hashFuncs * elements) / (data.length * 8)), hashFuncs);
+    }
+
+    @Override
+    public String toString() {
+        return "Bloom Filter of size " + data.length + " with " + hashFuncs + " hash functions.";
+    }
+
+    @Override
+    void parse() throws ProtocolException {
+        data = readByteArray();
+        if (data.length > MAX_FILTER_SIZE)
+            throw new ProtocolException ("Bloom filter out of size range.");
+        
+        hashFuncs = readUint32();
+        if (hashFuncs > MAX_HASH_FUNCS)
+            throw new ProtocolException("Bloom filter hash function count out of range");
+        
+        nTweak = readUint32();
+
+        length = cursor - offset;
+    }
+    
+    /**
+     * Serializes this message to the provided stream. If you just want the raw bytes use bitcoinSerialize().
+     */
+    void bitcoinSerializeToStream(OutputStream stream) throws IOException {
+        stream.write(new VarInt(data.length).encode());
+        stream.write(data);
+        Utils.uint32ToByteStreamLE(hashFuncs, stream);
+        Utils.uint32ToByteStreamLE(nTweak, stream);
+    }
+
+    @Override
+    protected void parseLite() throws ProtocolException {
+        // Do nothing, lazy parsing isn't useful for bloom filters.
+    }
+
+    private int ROTL32 (int x, int r) {
+      return (x << r) | (x >>> (32 - r));
+    }
+    
+    private int hash(int hashNum, byte[] object) {
+        // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
+        int h1 = (int)((hashNum * 0xFBA4C795L + nTweak) & 0xFFFFFFFF);
+        final int c1 = 0xcc9e2d51;
+        final int c2 = 0x1b873593;
+
+        int numBlocks = (object.length / 4) * 4;
+        // body
+        for(int i = 0; i < numBlocks; i += 4) {
+            int k1 = (object[i] & 0xFF) |
+                  ((object[i+1] & 0xFF) << 8) |
+                  ((object[i+2] & 0xFF) << 16) |
+                  ((object[i+3] & 0xFF) << 24);
+            
+            k1 *= c1;
+            k1 = ROTL32(k1,15);
+            k1 *= c2;
+
+            h1 ^= k1;
+            h1 = ROTL32(h1,13); 
+            h1 = h1*5+0xe6546b64;
+        }
+        
+        int k1 = 0;
+        switch(object.length & 3)
+        {
+        case 3: k1 ^= (object[numBlocks + 2] & 0xff) << 16;
+        case 2: k1 ^= (object[numBlocks + 1] & 0xff) << 8;
+        case 1: k1 ^= (object[numBlocks] & 0xff);
+                k1 *= c1; k1 = ROTL32(k1,15); k1 *= c2; h1 ^= k1;
+        };
+
+        // finalization
+        h1 ^= object.length;
+        h1 ^= h1 >>> 16;
+        h1 *= 0x85ebca6b;
+        h1 ^= h1 >>> 13;
+        h1 *= 0xc2b2ae35;
+        h1 ^= h1 >>> 16;
+        
+        return (int)((h1&0xFFFFFFFFL) % (data.length * 8));
+    }
+    
+    /**
+     * Returns true if the given object matches the filter
+     * (either because it was inserted, or because we have a false-positive)
+     */
+    public boolean contains(byte[] object) {
+        for (int i = 0; i < hashFuncs; i++) {
+            if (!Utils.checkBitLE(data, hash(i, object)))
+                return false;
+        }
+        return true;
+    }
+    
+    /**
+     * Insert the given arbitrary data into the filter
+     */
+    public void insert(byte[] object) {
+        for (int i = 0; i < hashFuncs; i++)
+            Utils.setBitLE(data, hash(i, object));
+    }
+
+    /**
+     * Copies filter into this.
+     * filter must have the same size, hash function count and nTweak or an exception will be thrown.
+     */
+    public void merge(BloomFilter filter) {
+        Preconditions.checkArgument(filter.data.length == this.data.length &&
+                filter.hashFuncs == this.hashFuncs &&
+                filter.nTweak == this.nTweak);
+        for (int i = 0; i < data.length; i++)
+            this.data[i] |= filter.data[i];
+    }
+    
+    @Override
+    public boolean equals(Object other) {
+        if (other instanceof BloomFilter &&
+                ((BloomFilter)other).hashFuncs == this.hashFuncs &&
+                ((BloomFilter)other).nTweak == this.nTweak &&
+                Arrays.equals(((BloomFilter)other).data, this.data))
+            return true;
+        return false;
+    }
+}

core/src/test/java/com/google/bitcoin/core/BloomFilterTest.java

@@ -0,0 +1,47 @@
+package com.google.bitcoin.core;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import org.junit.Test;
+import org.spongycastle.util.encoders.Hex;
+
+public class BloomFilterTest {
+    @Test
+    public void insertSerializeTest() {
+        BloomFilter filter = new BloomFilter(3, 0.01, 0);
+        
+        filter.insert(Hex.decode("99108ad8ed9bb6274d3980bab5a85c048f0950c8"));
+        assertTrue (filter.contains(Hex.decode("99108ad8ed9bb6274d3980bab5a85c048f0950c8")));
+        // One bit different in first byte
+        assertFalse(filter.contains(Hex.decode("19108ad8ed9bb6274d3980bab5a85c048f0950c8")));
+
+        filter.insert(Hex.decode("b5a2c786d9ef4658287ced5914b37a1b4aa32eee"));
+        assertTrue(filter.contains(Hex.decode("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")));
+        
+        filter.insert(Hex.decode("b9300670b4c5366e95b2699e8b18bc75e5f729c5"));
+        assertTrue(filter.contains(Hex.decode("b9300670b4c5366e95b2699e8b18bc75e5f729c5")));
+        
+        // Value generated by the reference client
+        assertTrue(Arrays.equals(Hex.decode("03614e9b0500000000000000"), filter.bitcoinSerialize()));
+    }
+    
+    @Test
+    public void insertSerializeTestWithTweak() {
+        BloomFilter filter = new BloomFilter(3, 0.01, 2147483649L);
+        
+        filter.insert(Hex.decode("99108ad8ed9bb6274d3980bab5a85c048f0950c8"));
+        assertTrue (filter.contains(Hex.decode("99108ad8ed9bb6274d3980bab5a85c048f0950c8")));
+        // One bit different in first byte
+        assertFalse(filter.contains(Hex.decode("19108ad8ed9bb6274d3980bab5a85c048f0950c8")));
+
+        filter.insert(Hex.decode("b5a2c786d9ef4658287ced5914b37a1b4aa32eee"));
+        assertTrue(filter.contains(Hex.decode("b5a2c786d9ef4658287ced5914b37a1b4aa32eee")));
+        
+        filter.insert(Hex.decode("b9300670b4c5366e95b2699e8b18bc75e5f729c5"));
+        assertTrue(filter.contains(Hex.decode("b9300670b4c5366e95b2699e8b18bc75e5f729c5")));
+        
+        // Value generated by the reference client
+        assertTrue(Arrays.equals(Hex.decode("03ce42990500000001000080"), filter.bitcoinSerialize()));
+    }
+}