Bloom filtering upgrades: can now create FilteredBlock's by applying a BloomFilter to a block. This is primarily intended for unit testing.

core/src/main/java/com/google/bitcoin/core/BloomFilter.java

@@ -16,11 +16,15 @@
 
 package com.google.bitcoin.core;
 
+import com.google.bitcoin.script.Script;
+import com.google.bitcoin.script.ScriptChunk;
 import com.google.common.base.Objects;
 
 import java.io.IOException;
 import java.io.OutputStream;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.List;
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static java.lang.Math.*;
@@ -157,9 +161,12 @@ public class BloomFilter extends Message {
     private static int rotateLeft32(int x, int r) {
         return (x << r) | (x >>> (32 - r));
     }
-    
-    private int hash(int hashNum, byte[] object) {
-        // The following is MurmurHash3 (x86_32), see http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp
+
+    /**
+     * Applies the MurmurHash3 (x86_32) algorithm to the given data.
+     * See this <a href="http://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp">C++ code for the original.</a>
+     */
+    public static int murmurHash3(byte[] data, long nTweak, int hashNum, byte[] object) {
         int h1 = (int)(hashNum * 0xFBA4C795L + nTweak);
         final int c1 = 0xcc9e2d51;
         final int c2 = 0x1b873593;
@@ -214,22 +221,22 @@ public class BloomFilter extends Message {
      * Returns true if the given object matches the filter either because it was inserted, or because we have a
      * false-positive.
      */
-    public boolean contains(byte[] object) {
+    public synchronized boolean contains(byte[] object) {
         for (int i = 0; i < hashFuncs; i++) {
-            if (!Utils.checkBitLE(data, hash(i, object)))
+            if (!Utils.checkBitLE(data, murmurHash3(data, nTweak, i, object)))
                 return false;
         }
         return true;
     }
     
     /** Insert the given arbitrary data into the filter */
-    public void insert(byte[] object) {
+    public synchronized void insert(byte[] object) {
         for (int i = 0; i < hashFuncs; i++)
-            Utils.setBitLE(data, hash(i, object));
+            Utils.setBitLE(data, murmurHash3(data, nTweak, i, object));
     }
 
     /** Inserts the given key and equivalent hashed form (for the address). */
-    public void insert(ECKey key) {
+    public synchronized void insert(ECKey key) {
         insert(key.getPubKey());
         insert(key.getPubKeyHash());
     }
@@ -241,7 +248,7 @@ public class BloomFilter extends Message {
      * Solved blocks will then be send just as Merkle trees of tx hashes, meaning a constant 32 bytes of data for each
      * transaction instead of 100-300 bytes as per usual.
      */
-    public void setMatchAll() {
+    public synchronized void setMatchAll() {
         data = new byte[] {(byte) 0xff};
     }
 
@@ -249,7 +256,7 @@ public class BloomFilter extends Message {
      * Copies filter into this. Filter must have the same size, hash function count and nTweak or an
      * IllegalArgumentException will be thrown.
      */
-    public void merge(BloomFilter filter) {
+    public synchronized void merge(BloomFilter filter) {
         if (!this.matchesAll() && !filter.matchesAll()) {
             checkArgument(filter.data.length == this.data.length &&
                           filter.hashFuncs == this.hashFuncs &&
@@ -265,15 +272,69 @@ public class BloomFilter extends Message {
      * Returns true if this filter will match anything. See {@link com.google.bitcoin.core.BloomFilter#setMatchAll()}
      * for when this can be a useful thing to do.
      */
-    public boolean matchesAll() {
+    public synchronized boolean matchesAll() {
         for (byte b : data)
             if (b != (byte) 0xff)
                 return false;
         return true;
     }
+
+    /**
+     * The update flag controls how application of the filter to a block modifies the filter. See the enum javadocs
+     * for information on what occurs and when.
+     */
+    public synchronized BloomUpdate getUpdateFlag() {
+        if (nFlags == 0)
+            return BloomUpdate.UPDATE_NONE;
+        else if (nFlags == 1)
+            return BloomUpdate.UPDATE_ALL;
+        else if (nFlags == 2)
+            return BloomUpdate.UPDATE_P2PUBKEY_ONLY;
+        else
+            throw new IllegalStateException("Unknown flag combination");
+    }
+
+    /**
+     * Creates a new FilteredBlock from the given Block, using this filter to select transactions. Matches can cause the
+     * filter to be updated with the matched element, this ensures that when a filter is applied to a block, spends of
+     * matched transactions are also matched. However it means this filter can be mutated by the operation.
+     */
+    public synchronized FilteredBlock applyAndUpdate(Block block) {
+        List<Transaction> txns = block.getTransactions();
+        List<Sha256Hash> txHashes = new ArrayList<Sha256Hash>(txns.size());
+        byte[] bits = new byte[(int) Math.ceil(txns.size() / 8.0)];
+        for (int i = 0; i < txns.size(); i++) {
+            txHashes.add(txns.get(i).getHash());
+            if (applyAndUpdate(txns.get(i)))
+                Utils.setBitLE(bits, i);
+        }
+        PartialMerkleTree pmt = PartialMerkleTree.buildFromLeaves(block.getParams(), bits, txHashes);
+        return new FilteredBlock(block.getParams(), block.cloneAsHeader(), pmt);
+    }
+
+    public synchronized boolean applyAndUpdate(Transaction tx) {
+        if (contains(tx.getHash().getBytes()))
+            return true;
+        boolean found = false;
+        BloomUpdate flag = getUpdateFlag();
+        for (TransactionOutput output : tx.getOutputs()) {
+            Script script = output.getScriptPubKey();
+            for (ScriptChunk chunk : script.getChunks()) {
+                if (!chunk.isPushData())
+                    continue;
+                if (contains(chunk.data)) {
+                    boolean isSendingToPubKeys = script.isSentToRawPubKey() || script.isSentToMultiSig();
+                    if (flag == BloomUpdate.UPDATE_ALL || (flag == BloomUpdate.UPDATE_P2PUBKEY_ONLY && isSendingToPubKeys))
+                        insert(output.getOutPointFor().bitcoinSerialize());
+                    found = true;
+                }
+            }
+        }
+        return found;
+    }
     
     @Override
-    public boolean equals(Object o) {
+    public synchronized boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         BloomFilter other = (BloomFilter) o;
@@ -283,7 +344,7 @@ public class BloomFilter extends Message {
     }
 
     @Override
-    public int hashCode() {
+    public synchronized int hashCode() {
         return Objects.hashCode(hashFuncs, nTweak, Arrays.hashCode(data));
     }
 }

core/src/main/java/com/google/bitcoin/core/FilteredBlock.java

@@ -29,7 +29,6 @@ public class FilteredBlock extends Message {
     public static final int MIN_PROTOCOL_VERSION = 70000;
     private Block header;
 
-    // The PartialMerkleTree of transactions
     private PartialMerkleTree merkleTree;
     private List<Sha256Hash> cachedTransactionHashes = null;
     
@@ -40,7 +39,13 @@ public class FilteredBlock extends Message {
     public FilteredBlock(NetworkParameters params, byte[] payloadBytes) throws ProtocolException {
         super(params, payloadBytes, 0);
     }
-    
+
+    public FilteredBlock(NetworkParameters params, Block header, PartialMerkleTree pmt) {
+        super(params);
+        this.header = header;
+        this.merkleTree = pmt;
+    }
+
     @Override
     public void bitcoinSerializeToStream(OutputStream stream) throws IOException {
         if (header.transactions == null)

core/src/main/java/com/google/bitcoin/core/PartialMerkleTree.java

@@ -23,6 +23,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import static com.google.bitcoin.core.Utils.*;
+
 /**
  * <p>A data structure that contains proofs of block inclusion for one or more transactions, in an efficient manner.</p>
  *
@@ -62,14 +64,44 @@ public class PartialMerkleTree extends Message {
     public PartialMerkleTree(NetworkParameters params, byte[] payloadBytes, int offset) throws ProtocolException {
         super(params, payloadBytes, offset);
     }
-    
+
+    /**
+     * Constructs a new PMT with the given bit set (little endian) and the raw list of hashes including internal hashes,
+     * taking ownership of the list.
+     */
+    public PartialMerkleTree(NetworkParameters params, byte[] bits, List<Sha256Hash> hashes, int origTxCount) {
+        super(params);
+        this.matchedChildBits = bits;
+        this.hashes = hashes;
+        this.transactionCount = origTxCount;
+    }
+
+    /**
+     * Calculates a PMT given the list of leaf hashes and which leaves need to be included. The relevant interior hashes
+     * are calculated and a new PMT returned.
+     */
+    public static PartialMerkleTree buildFromLeaves(NetworkParameters params, byte[] includeBits, List<Sha256Hash> allLeafHashes) {
+        // Calculate height of the tree.
+        int height = 0;
+        while (getTreeWidth(allLeafHashes.size(), height) > 1)
+            height++;
+        List<Boolean> bitList = new ArrayList<Boolean>();
+        List<Sha256Hash> hashes = new ArrayList<Sha256Hash>();
+        traverseAndBuild(height, 0, allLeafHashes, includeBits, bitList, hashes);
+        byte[] bits = new byte[(int)Math.ceil(bitList.size() / 8.0)];
+        for (int i = 0; i < bitList.size(); i++)
+            if (bitList.get(i))
+                Utils.setBitLE(bits, i);
+        return new PartialMerkleTree(params, bits, hashes, allLeafHashes.size());
+    }
+
     @Override
     public void bitcoinSerializeToStream(OutputStream stream) throws IOException {
-        Utils.uint32ToByteStreamLE(transactionCount, stream);
+        uint32ToByteStreamLE(transactionCount, stream);
 
         stream.write(new VarInt(hashes.size()).encode());
         for (Sha256Hash hash : hashes)
-            stream.write(Utils.reverseBytes(hash.getBytes()));
+            stream.write(reverseBytes(hash.getBytes()));
 
         stream.write(new VarInt(matchedChildBits.length).encode());
         stream.write(matchedChildBits);
@@ -86,18 +118,62 @@ public class PartialMerkleTree extends Message {
 
         int nFlagBytes = (int) readVarInt();
         matchedChildBits = readBytes(nFlagBytes);
-        
+
         length = cursor - offset;
     }
-    
+
+    // Based on CPartialMerkleTree::TraverseAndBuild in Bitcoin Core.
+    private static void traverseAndBuild(int height, int pos, List<Sha256Hash> allLeafHashes, byte[] includeBits,
+                                         List<Boolean> matchedChildBits, List<Sha256Hash> resultHashes) {
+        boolean parentOfMatch = false;
+        // Is this node a parent of at least one matched hash?
+        for (int p = pos << height; p < (pos+1) << height && p < allLeafHashes.size(); p++) {
+            if (Utils.checkBitLE(includeBits, p)) {
+                parentOfMatch = true;
+                break;
+            }
+        }
+        // Store as a flag bit.
+        matchedChildBits.add(parentOfMatch);
+        if (height == 0 || !parentOfMatch) {
+            // If at height 0, or nothing interesting below, store hash and stop.
+            resultHashes.add(calcHash(height, pos, allLeafHashes));
+        } else {
+            // Otherwise descend into the subtrees.
+            int h = height - 1;
+            int p = pos * 2;
+            traverseAndBuild(h, p, allLeafHashes, includeBits, matchedChildBits, resultHashes);
+            if (p + 1 < getTreeWidth(allLeafHashes.size(), h))
+                traverseAndBuild(h, p + 1, allLeafHashes, includeBits, matchedChildBits, resultHashes);
+        }
+    }
+
+    private static Sha256Hash calcHash(int height, int pos, List<Sha256Hash> hashes) {
+        if (height == 0) {
+            // Hash at height 0 is just the regular tx hash itself.
+            return hashes.get(pos);
+        }
+        int h = height - 1;
+        int p = pos * 2;
+        Sha256Hash left = calcHash(h, p, hashes);
+        // Calculate right hash if not beyond the end of the array - copy left hash otherwise.
+        Sha256Hash right;
+        if (p + 1 < getTreeWidth(hashes.size(), h)) {
+            right = calcHash(h, p + 1, hashes);
+        } else {
+            right = left;
+        }
+        return combineLeftRight(left.getBytes(), right.getBytes());
+    }
+
     @Override
     protected void parseLite() {
         
     }
     
     // helper function to efficiently calculate the number of nodes at given height in the merkle tree
-    private int getTreeWidth(int height) {
-        return (transactionCount+(1 << height)-1) >> height;
+    private static int getTreeWidth(int transactionCount, int height) {
+        return (transactionCount + (1 << height) - 1) >> height;
     }
     
     private static class ValuesUsed {
@@ -111,30 +187,35 @@ public class PartialMerkleTree extends Message {
             // overflowed the bits array - failure
             throw new VerificationException("CPartialMerkleTree overflowed its bits array");
         }
-        boolean parentOfMatch = Utils.checkBitLE(matchedChildBits, used.bitsUsed++);
+        boolean parentOfMatch = checkBitLE(matchedChildBits, used.bitsUsed++);
         if (height == 0 || !parentOfMatch) {
             // if at height 0, or nothing interesting below, use stored hash and do not descend
             if (used.hashesUsed >= hashes.size()) {
                 // overflowed the hash array - failure
                 throw new VerificationException("CPartialMerkleTree overflowed its hash array");
             }
+            Sha256Hash hash = hashes.get(used.hashesUsed++);
             if (height == 0 && parentOfMatch) // in case of height 0, we have a matched txid
-                matchedHashes.add(hashes.get(used.hashesUsed));
-            return hashes.get(used.hashesUsed++);
+                matchedHashes.add(hash);
+            return hash;
         } else {
             // otherwise, descend into the subtrees to extract matched txids and hashes
-            byte[] left = recursiveExtractHashes(height-1, pos*2, used, matchedHashes).getBytes(), right;
-            if (pos*2+1 < getTreeWidth(height-1))
-                right = recursiveExtractHashes(height-1, pos*2+1, used, matchedHashes).getBytes();
+            byte[] left = recursiveExtractHashes(height - 1, pos * 2, used, matchedHashes).getBytes(), right;
+            if (pos * 2 + 1 < getTreeWidth(transactionCount, height-1))
+                right = recursiveExtractHashes(height - 1, pos * 2 + 1, used, matchedHashes).getBytes();
             else
                 right = left;
             // and combine them before returning
-            return new Sha256Hash(Utils.reverseBytes(Utils.doubleDigestTwoBuffers(
-                    Utils.reverseBytes(left), 0, 32,
-                    Utils.reverseBytes(right), 0, 32)));
+            return combineLeftRight(left, right);
         }
     }
-    
+
+    private static Sha256Hash combineLeftRight(byte[] left, byte[] right) {
+        return new Sha256Hash(reverseBytes(doubleDigestTwoBuffers(
+                reverseBytes(left), 0, 32,
+                reverseBytes(right), 0, 32)));
+    }
+
     /**
      * Extracts tx hashes that are in this merkle tree
      * and returns the merkle root of this tree.
@@ -164,7 +245,7 @@ public class PartialMerkleTree extends Message {
             throw new VerificationException("Got a CPartialMerkleTree with fewer matched bits than hashes");
         // calculate height of tree
         int height = 0;
-        while (getTreeWidth(height) > 1)
+        while (getTreeWidth(transactionCount, height) > 1)
             height++;
         // traverse the partial tree
         ValuesUsed used = new ValuesUsed();

core/src/main/java/com/google/bitcoin/core/PeerGroup.java

@@ -143,7 +143,9 @@ public class PeerGroup extends AbstractExecutionThreadService implements Transac
             final double rate = checkNotNull(chain).getFalsePositiveRate();
             final double target = bloomFilterMerger.getBloomFilterFPRate() * MAX_FP_RATE_INCREASE;
             if (rate > target) {
-                log.info("Force update Bloom filter due to high false positive rate ({} vs {})", rate, target);
+                // TODO: Avoid hitting this path if the remote peer didn't acknowledge applying a new filter yet.
+                if (log.isDebugEnabled())
+                    log.debug("Force update Bloom filter due to high false positive rate ({} vs {})", rate, target);
                 recalculateFastCatchupAndFilter(FilterRecalculateMode.FORCE_SEND_FOR_REFRESH);
             }
         }

core/src/main/java/com/google/bitcoin/testing/FakeTxBuilder.java

@@ -208,4 +208,14 @@ public class FakeTxBuilder {
         b.solve();
         return b;
     }
+
+    public static Block makeSolvedTestBlock(Block prev, Address to, Transaction... transactions) throws BlockStoreException {
+        Block b = prev.createNextBlock(to);
+        // Coinbase tx already exists.
+        for (Transaction tx : transactions) {
+            b.addTransaction(tx);
+        }
+        b.solve();
+        return b;
+    }
 }

core/src/test/java/com/google/bitcoin/core/FilteredBlockAndPartialMerkleTreeTests.java

@@ -20,6 +20,7 @@ package com.google.bitcoin.core;
 import com.google.bitcoin.core.TransactionConfidence.ConfidenceType;
 import com.google.bitcoin.params.UnitTestParams;
 import com.google.bitcoin.store.MemoryBlockStore;
+import com.google.bitcoin.testing.FakeTxBuilder;
 import com.google.bitcoin.testing.InboundMessageQueuer;
 import com.google.bitcoin.testing.TestWithPeerGroup;
 import com.google.bitcoin.wallet.KeyChainGroup;
@@ -68,7 +69,25 @@ public class FilteredBlockAndPartialMerkleTreeTests extends TestWithPeerGroup {
         // Check round tripping.
         assertEquals(block, new FilteredBlock(params, block.bitcoinSerialize()));
     }
-    
+
+    @Test
+    public void createFilteredBlock() throws Exception {
+        ECKey key1 = new ECKey();
+        ECKey key2 = new ECKey();
+        Transaction tx1 = FakeTxBuilder.createFakeTx(params, Coin.COIN,  key1);
+        Transaction tx2 = FakeTxBuilder.createFakeTx(params, Coin.FIFTY_COINS, key2.toAddress(params));
+        Block block = FakeTxBuilder.makeSolvedTestBlock(params.getGenesisBlock(), new Address(params, "msg2t2V2sWNd85LccoddtWysBTR8oPnkzW"), tx1, tx2);
+        BloomFilter filter = new BloomFilter(4, 0.1, 1);
+        filter.insert(key1);
+        filter.insert(key2);
+        FilteredBlock filteredBlock = filter.applyAndUpdate(block);
+        assertEquals(4, filteredBlock.getTransactionCount());
+        // This call triggers verification of the just created data.
+        List<Sha256Hash> txns = filteredBlock.getTransactionHashes();
+        assertTrue(txns.contains(tx1.getHash()));
+        assertTrue(txns.contains(tx2.getHash()));
+    }
+
     @Test
     public void serializeDownloadBlockWithWallet() throws Exception {
         unitTestParams = UnitTestParams.get();