HD Wallets: fix key lookahead and auto-advance so it works as intended.

Previously the codepath that was supposed to mark keys as used didn't work, and the lookahead calculation wasn't quite right. Now the current key advances correctly when an inbound tx is found that spends to it, including pending transactions. Additionally the lookahead zone now has the threshold zone after it, not inside it, meaning that if you request a lookahead size of 100 keys you'll actually always have at least 100 keys, never less.

HD Wallets TODO.txt

@@ -2,7 +2,6 @@
 - Calculate lookahead keys on a background thread.
 - Redo internals of DKC to support arbitrary tree structures.
 - Add a REFUND key purpose and map to the receive tree (for now).
-- Write unit tests for auto key lookahead code from Harold.
 - Pre-check the keys for all attached BlockChainListeners and reset the chain download if any of them throw an exception
   then use this to ensure that we can't accidentally run off the end of an HDW heirarchy when downloading the chain.
 

core/src/main/java/com/google/bitcoin/core/Wallet.java

@@ -1816,7 +1816,9 @@ public class Wallet extends BaseTaggableObject implements Serializable, BlockCha
             tx.getConfidence().setConfidenceType(ConfidenceType.PENDING);
             confidenceChanged.put(tx, TransactionConfidence.Listener.ChangeReason.TYPE);
             addWalletTransaction(Pool.PENDING, tx);
-
+            // Mark any keys used in the outputs as "used", this allows wallet UI's to auto-advance the current key
+            // they are showing to the user in qr codes etc.
+            markKeysAsUsed(tx);
             try {
                 Coin valueSentFromMe = tx.getValueSentFromMe(this);
                 Coin valueSentToMe = tx.getValueSentToMe(this);

core/src/main/java/com/google/bitcoin/wallet/DeterministicKeyChain.java

@@ -22,8 +22,6 @@ import com.google.bitcoin.core.Utils;
 import com.google.bitcoin.crypto.*;
 import com.google.bitcoin.store.UnreadableWalletException;
 import com.google.bitcoin.utils.Threading;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
 import com.google.common.collect.ImmutableList;
 import com.google.protobuf.ByteString;
 import org.bitcoinj.wallet.Protos;
@@ -33,8 +31,6 @@ import org.spongycastle.crypto.params.KeyParameter;
 import org.spongycastle.math.ec.ECPoint;
 
 import javax.annotation.Nullable;
-
-import java.io.UnsupportedEncodingException;
 import java.math.BigInteger;
 import java.security.SecureRandom;
 import java.util.ArrayList;
@@ -77,6 +73,23 @@ import static com.google.common.collect.Lists.newLinkedList;
  * and finally the actual leaf keys that users use hanging off the end. The leaf keys are special in that they don't
  * internally store the private part at all, instead choosing to rederive the private key from the parent when
  * needed for signing. This simplifies the design for encrypted key chains.</p>
+ *
+ * <p>The key chain manages a <i>lookahead zone</i>. This zone is required because when scanning the chain, you don't
+ * know exactly which keys might receive payments. The user may have handed out several addresses and received payments
+ * on them, but for latency reasons the block chain is requested from remote peers in bulk, meaning you must
+ * "look ahead" when calculating keys to put in the Bloom filter. The default lookahead zone is 100 keys, meaning if
+ * the user hands out more than 100 addresses and receives payment on them before the chain is next scanned, some
+ * transactions might be missed. 100 is a reasonable choice for consumer wallets running on CPU constrained devices.
+ * For industrial wallets that are receiving keys all the time, a higher value is more appropriate. Ideally DKC and the
+ * wallet would know how to adjust this value automatically, but that's not implemented at the moment.</p>
+ *
+ * <p>In fact the real size of the lookahead zone is larger than requested, by default, it's one third larger. This
+ * is because the act of deriving new keys means recalculating the Bloom filters and this is an expensive operation.
+ * Thus, to ensure we don't have to recalculate on every single new key/address requested or seen we add more buffer
+ * space and only extend the lookahead zone when that buffer is exhausted. For example with a lookahead zone of 100
+ * keys, you can request 33 keys before more keys will be calculated and the Bloom filter rebuilt and rebroadcast.
+ * But even when you are requesting the 33rd key, you will still be looking 100 keys ahead.
+ * </p>
  */
 public class DeterministicKeyChain implements EncryptableKeyChain {
     private static final Logger log = LoggerFactory.getLogger(DeterministicKeyChain.class);
@@ -109,7 +122,11 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
     // The lookahead threshold causes us to batch up creation of new keys to minimize the frequency of Bloom filter
     // regenerations, which are expensive and will (in future) trigger chain download stalls/retries. One third
     // is an efficiency tradeoff.
-    private int lookaheadThreshold = lookaheadSize / 3;
+    private int lookaheadThreshold = calcDefaultLookaheadThreshold();
+
+    private int calcDefaultLookaheadThreshold() {
+        return lookaheadSize / 3;
+    }
 
     // The parent keys for external keys (handed out to other people) and internal keys (used for change addresses).
     private DeterministicKey externalKey, internalKey;
@@ -307,7 +324,7 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
 
     /** Returns freshly derived key/s that have not been returned by this method before. */
     @Override
-    public List<DeterministicKey> getKeys(KeyPurpose purpose,int numberOfKeys) {
+    public List<DeterministicKey> getKeys(KeyPurpose purpose, int numberOfKeys) {
         checkArgument(numberOfKeys > 0);
         lock.lock();
         try {
@@ -396,13 +413,14 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
      * Mark the DeterministicKeys as used, if they match the pubkeyHash
      * See {@link com.google.bitcoin.wallet.DeterministicKeyChain#markKeyAsUsed(DeterministicKey)} for more info on this.
      */
-    public boolean markPubHashAsUsed(byte[] pubkeyHash) {
+    @Nullable
+    public DeterministicKey markPubHashAsUsed(byte[] pubkeyHash) {
         lock.lock();
         try {
             DeterministicKey k = (DeterministicKey) basicKeyChain.findKeyFromPubHash(pubkeyHash);
             if (k != null)
                 markKeyAsUsed(k);
-            return k != null;
+            return k;
         } finally {
             lock.unlock();
         }
@@ -412,13 +430,14 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
      * Mark the DeterministicKeys as used, if they match the pubkey
      * See {@link com.google.bitcoin.wallet.DeterministicKeyChain#markKeyAsUsed(DeterministicKey)} for more info on this.
      */
-    public boolean markPubKeyAsUsed(byte[] pubkey) {
+    @Nullable
+    public DeterministicKey markPubKeyAsUsed(byte[] pubkey) {
         lock.lock();
         try {
             DeterministicKey k = (DeterministicKey) basicKeyChain.findKeyFromPubKey(pubkey);
             if (k != null)
                 markKeyAsUsed(k);
-            return k != null;
+            return k;
         } finally {
             lock.unlock();
         }
@@ -820,23 +839,25 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
      * Sets a new lookahead size. See {@link #getLookaheadSize()} for details on what this is. Setting a new size
      * that's larger than the current size will return immediately and the new size will only take effect next time
      * a fresh filter is requested (e.g. due to a new peer being connected). So you should set this before starting
-     * to sync the chain, if you want to modify it.
+     * to sync the chain, if you want to modify it. If you haven't modified the lookahead threshold manually then
+     * it will be automatically set to be a third of the new size.
      */
     public void setLookaheadSize(int lookaheadSize) {
         lock.lock();
         try {
+            boolean readjustThreshold = this.lookaheadThreshold == calcDefaultLookaheadThreshold();
             this.lookaheadSize = lookaheadSize;
+            if (readjustThreshold)
+                this.lookaheadThreshold = calcDefaultLookaheadThreshold();
         } finally {
             lock.unlock();
         }
     }
 
     /**
-     * Sets the threshold for the key pre-generation.
+     * Sets the threshold for the key pre-generation. This is used to avoid adding new keys and thus
-     * If a key is used in a transaction, the keychain would pre-generate a new key, for every issued key,
+     * re-calculating Bloom filters every time a new key is calculated. Without a lookahead threshold, every time we
-     * even if it is only one. If the blockchain is replayed, every key would trigger a regeneration
+     * received a relevant transaction we'd extend the lookahead zone and generate a new filter, which is inefficient.
-     * of the bloom filter sent to the peers as a consequence.
-     * To prevent this, new keys are only generated, if more than the threshold value are needed.
      */
     public void setLookaheadThreshold(int num) {
         lock.lock();
@@ -883,24 +904,24 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
      * Pre-generate enough keys to reach the lookahead size, but only if there are more than the lookaheadThreshold to
      * be generated, so that the Bloom filter does not have to be regenerated that often.
      *
-     * The return mutable list of keys must be inserted into the basic key chain.
+     * The returned mutable list of keys must be inserted into the basic key chain.
      */
     private List<DeterministicKey> maybeLookAhead(DeterministicKey parent, int issued) {
         checkState(lock.isHeldByCurrentThread());
         final int numChildren = hierarchy.getNumChildren(parent.getPath());
-        final int needed = issued + getLookaheadSize() - numChildren;
+        final int lookaheadSize = getLookaheadSize();
+        final int lookaheadThreshold = getLookaheadThreshold();
+        final int needed = issued + lookaheadSize + lookaheadThreshold - numChildren;
 
-        log.info("maybeLookAhead(): {} needed = lookaheadSize({}) - (numChildren({}) - issued({}) = {} < lookaheadThreshold({}))",
+        log.info("{} keys needed = {} issued + {} lookahead size + {} lookahead threshold - {} num children",
-                parent.getPathAsString(), getLookaheadSize(), numChildren,
+                needed, issued, lookaheadSize, lookaheadThreshold, numChildren);
-                issued, needed, getLookaheadThreshold());
 
-        /* Even if needed is negative, we have more than enough */
+        if (needed <= lookaheadThreshold)
-        if (needed <= getLookaheadThreshold())
             return new ArrayList<DeterministicKey>();
 
         List<DeterministicKey> result  = new ArrayList<DeterministicKey>(needed);
         long now = System.currentTimeMillis();
-        log.info("maybeLookAhead(): Pre-generating {} keys for {}", needed, parent.getPathAsString());
+        log.info("Pre-generating {} keys for {}", needed, parent.getPathAsString());
         for (int i = 0; i < needed; i++) {
             // TODO: Handle the case where the derived key is >= curve order.
             DeterministicKey key = HDKeyDerivation.deriveChildKey(parent, numChildren + i);
@@ -908,7 +929,7 @@ public class DeterministicKeyChain implements EncryptableKeyChain {
             hierarchy.putKey(key);
             result.add(key);
         }
-        log.info("maybeLookAhead(): Took {} msec", System.currentTimeMillis() - now);
+        log.info("Took {} msec", System.currentTimeMillis() - now);
         return result;
     }
 

core/src/main/java/com/google/bitcoin/wallet/KeyChainGroup.java

@@ -59,6 +59,9 @@ import static com.google.common.base.Preconditions.*;
  * <p>The wallet delegates most key management tasks to this class. It is <b>not</b> thread safe and requires external
  * locking, i.e. by the wallet lock. The group then in turn delegates most operations to the key chain objects,
  * combining their responses together when necessary.</p>
+ *
+ * <p>Deterministic key chains have a concept of a lookahead size and threshold. Please see the discussion in the
+ * class docs for {@link com.google.bitcoin.wallet.DeterministicKeyChain} for more information on this topic.</p>
  */
 public class KeyChainGroup {
     private static final Logger log = LoggerFactory.getLogger(KeyChainGroup.class);
@@ -129,9 +132,10 @@ public class KeyChainGroup {
         for (DeterministicKey key : followingAccountKeys) {
             checkArgument(key.getPath().size() == 1, "Following keys have to be account keys");
             DeterministicKeyChain chain = DeterministicKeyChain.watchAndFollow(key);
-            if (lookaheadSize > 0) {
+            if (lookaheadSize >= 0)
                 chain.setLookaheadSize(lookaheadSize);
-            }
+            if (lookaheadThreshold >= 0)
+                chain.setLookaheadThreshold(lookaheadThreshold);
             followingKeychains.put(accountKey, chain);
         }
     }
@@ -347,6 +351,9 @@ public class KeyChainGroup {
      * for more information.
      */
     public int getLookaheadSize() {
+        if (lookaheadSize == -1)
+            return getActiveKeyChain().getLookaheadSize();
+        else
             return lookaheadSize;
     }
 
@@ -367,6 +374,9 @@ public class KeyChainGroup {
      * for more information.
      */
     public int getLookaheadThreshold() {
+        if (lookaheadThreshold == -1)
+            return getActiveKeyChain().getLookaheadThreshold();
+        else
             return lookaheadThreshold;
     }
 
@@ -432,10 +442,23 @@ public class KeyChainGroup {
      */
     public void markPubKeyHashAsUsed(byte[] pubkeyHash) {
         for (DeterministicKeyChain chain : chains) {
-            if (chain.markPubHashAsUsed(pubkeyHash))
+            DeterministicKey key;
+            if ((key = chain.markPubHashAsUsed(pubkeyHash)) != null) {
+                markKeyAsUsed(key);
                 return;
             }
         }
+    }
+
+    /** If the given key is "current", advance the current key to a new one. */
+    private void markKeyAsUsed(DeterministicKey key) {
+        for (Map.Entry<KeyChain.KeyPurpose, DeterministicKey> entry : currentKeys.entrySet()) {
+            if (entry.getValue().equals(key)) {
+                log.info("Marking key as used: {}", key);
+                freshKey(entry.getKey());
+            }
+        }
+    }
 
 
     public boolean hasKey(ECKey key) {
@@ -465,10 +488,13 @@ public class KeyChainGroup {
      */
     public void markPubKeyAsUsed(byte[] pubkey) {
         for (DeterministicKeyChain chain : chains) {
-            if (chain.markPubKeyAsUsed(pubkey))
+            DeterministicKey key;
+            if ((key = chain.markPubKeyAsUsed(pubkey)) != null) {
+                markKeyAsUsed(key);
                 return;
             }
         }
+    }
 
     /** Returns the number of keys managed by this group, including the lookahead buffers. */
     public int numKeys() {

core/src/test/java/com/google/bitcoin/core/PeerGroupTest.java

@@ -43,8 +43,8 @@ import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
-import static com.google.bitcoin.core.Coin.*;
+import static com.google.bitcoin.core.Coin.COIN;
-import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.bitcoin.core.Coin.valueOf;
 import static org.junit.Assert.*;
 
 
@@ -568,7 +568,9 @@ public class PeerGroupTest extends TestWithPeerGroup {
 
     @Test
     public void testBloomResendOnNewKey() throws Exception {
-        // Check that when we add a new key to the wallet, the Bloom filter is re-calculated and re-sent.
+        // Check that when we add a new key to the wallet, the Bloom filter is re-calculated and re-sent but only once
+        // we exceed the lookahead threshold.
+        wallet.setKeychainLookaheadSize(20);
         peerGroup.startAsync();
         peerGroup.awaitRunning();
         // Create a couple of peers.
@@ -576,36 +578,26 @@ public class PeerGroupTest extends TestWithPeerGroup {
         InboundMessageQueuer p2 = connectPeer(2);
         peerGroup.waitForJobQueue();
         BloomFilter f1 = p1.lastReceivedFilter;
-        BloomFilter f2 = p2.lastReceivedFilter;
+        int threshold = wallet.getKeychainLookaheadThreshold();
+        wallet.freshReceiveKey();  // Force generation with the new lookahead size.
+        peerGroup.waitForJobQueue();
+        assertEquals(BloomFilter.class, outbound(p1).getClass());
+        assertEquals(MemoryPoolMessage.class, outbound(p1).getClass());
         ECKey key = null;
-        // We have to run ahead of the lookahead zone for this test.
+        // We have to run ahead of the lookahead zone for this test. There should only be one bloom filter recalc.
-        for (int i = 0; i < wallet.getKeychainLookaheadSize() + 1; i++) {
+        for (int i = 0; i < threshold + 2; i++) {
             key = wallet.freshReceiveKey();
+        }
         // Wait here. Bloom filters are recalculated asynchronously so if we didn't wait, we might not pass the
         // test below where we expect each key to generate a new filter because this thread could generate all
         // the keys before the peergroup thread does the recalculation, causing only one filter to be sent.
         peerGroup.waitForJobQueue();
-        }
+        BloomFilter f3 = (BloomFilter) outbound(p1);
-        BloomFilter f3 = null;
-        BloomFilter f4 = null;
-        // Each time we request a fresh key, a new filter is sent. That's because the lookahead buffer is NOT an
-        // optimisation (currently), but rather is intended to try and ensure we don't miss transactions when
-        // catching up through the chain.
-        for (int i = 0; i < wallet.getKeychainLookaheadSize(); i++) {
-            f3 = (BloomFilter) outbound(p1);
         assertNotNull(f3);
         assertEquals(MemoryPoolMessage.class, outbound(p1).getClass());
-            f4 = (BloomFilter) outbound(p2);
+        assertNull(outbound(p1));
-            assertNotNull(f4);
-            assertEquals(MemoryPoolMessage.class, outbound(p2).getClass());
-        }
-        checkNotNull(f3);
-        checkNotNull(f4);
-        checkNotNull(key);
         // Check the last filter received.
         assertNotEquals(f1, f3);
-        assertNotEquals(f2, f4);
-        assertEquals(f3, f4);
         assertTrue(f3.contains(key.getPubKey()));
         assertTrue(f3.contains(key.getPubKeyHash()));
         assertFalse(f1.contains(key.getPubKey()));

core/src/test/java/com/google/bitcoin/core/WalletTest.java

@@ -2443,7 +2443,7 @@ public class WalletTest extends TestWithWallet {
             }
         }, Threading.SAME_THREAD);
         wallet.freshReceiveKey();
-        assertEquals(6, keys.size());
+        assertEquals(7, keys.size());
     }
 
     @Test

core/src/test/java/com/google/bitcoin/wallet/DeterministicKeyChainTest.java

@@ -89,23 +89,17 @@ public class DeterministicKeyChainTest {
         ECKey key = chain.getKey(KeyChain.KeyPurpose.CHANGE);
         assertEquals(1, listenerKeys.size());  // 1 event
         final List<ECKey> firstEvent = listenerKeys.get(0);
-        assertEquals(6, firstEvent.size());  // 5 lookahead keys and 1 to satisfy the request.
+        assertEquals(7, firstEvent.size());  // 5 lookahead keys, +1 lookahead threhsold, +1 to satisfy the request.
         assertTrue(firstEvent.contains(key));   // order is not specified.
         listenerKeys.clear();
-        key = chain.getKey(KeyChain.KeyPurpose.CHANGE);
+        chain.getKey(KeyChain.KeyPurpose.CHANGE);
+        // At this point we've entered the threshold zone so more keys won't immediately trigger more generations.
+        assertEquals(0, listenerKeys.size());  // 1 event
+        final int lookaheadThreshold = chain.getLookaheadThreshold();
+        for (int i = 0; i < lookaheadThreshold; i++)
+            chain.getKey(KeyChain.KeyPurpose.CHANGE);
         assertEquals(1, listenerKeys.size());  // 1 event
-        assertEquals(1, listenerKeys.get(0).size());  // 1 key.
+        assertEquals(lookaheadThreshold + 1, listenerKeys.get(0).size());  // 1 key.
-        DeterministicKey eventKey = (DeterministicKey) listenerKeys.get(0).get(0);
-        assertNotEquals(key, eventKey);  // The key added is not the one that's served.
-        assertEquals(6, eventKey.getChildNumber().i());
-        listenerKeys.clear();
-        key = chain.getKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
-        assertEquals(1, listenerKeys.size());  // 1 event
-        assertEquals(6, listenerKeys.get(0).size());  // 1 key.
-        eventKey = (DeterministicKey) listenerKeys.get(0).get(0);
-        // The key added IS the one that's served because we did not previously request any RECEIVE_FUNDS keys.
-        assertEquals(key, eventKey);
-        assertEquals(0, eventKey.getChildNumber().i());
     }
 
     @Test
@@ -125,8 +119,8 @@ public class DeterministicKeyChainTest {
         DeterministicKey key3 = chain.getKey(KeyChain.KeyPurpose.CHANGE);
 
         List<Protos.Key> keys = chain.serializeToProtobuf();
-        // 1 root seed, 1 master key, 1 account key, 2 internal keys, 3 derived and 20 lookahead.
+        // 1 root seed, 1 master key, 1 account key, 2 internal keys, 3 derived, 20 lookahead and 5 lookahead threshold.
-        assertEquals(28, keys.size());
+        assertEquals(33, keys.size());
 
         // Get another key that will be lost during round-tripping, to ensure we can derive it again.
         DeterministicKey key4 = chain.getKey(KeyChain.KeyPurpose.CHANGE);
@@ -253,12 +247,12 @@ public class DeterministicKeyChainTest {
 
     @Test
     public void bloom() {
-        DeterministicKey key1 = chain.getKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         DeterministicKey key2 = chain.getKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
+        DeterministicKey key1 = chain.getKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         // The filter includes the internal keys as well (for now), although I'm not sure if we should allow funds to
         // be received on them or not ....
-        assertEquals(32, chain.numBloomFilterEntries());
+        assertEquals(36, chain.numBloomFilterEntries());
-        BloomFilter filter = chain.getFilter(32, 0.001, 1);
+        BloomFilter filter = chain.getFilter(36, 0.001, 1);
         assertTrue(filter.contains(key1.getPubKey()));
         assertTrue(filter.contains(key1.getPubKeyHash()));
         assertTrue(filter.contains(key2.getPubKey()));

core/src/test/java/com/google/bitcoin/wallet/KeyChainGroupTest.java

@@ -69,7 +69,7 @@ public class KeyChainGroupTest {
         assertEquals(INITIAL_KEYS, group.numKeys());
         assertEquals(2 * INITIAL_KEYS, group.getBloomFilterElementCount());
         ECKey r1 = group.currentKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
-        final int keys = INITIAL_KEYS + LOOKAHEAD_SIZE + 1;
+        final int keys = INITIAL_KEYS + LOOKAHEAD_SIZE + group.getLookaheadThreshold() + 1;
         assertEquals(keys, group.numKeys());
         assertEquals(2 * keys, group.getBloomFilterElementCount());
 
@@ -152,12 +152,12 @@ public class KeyChainGroupTest {
 
         assertEquals(INITIAL_KEYS, group.numKeys());
         Address a1 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
-        assertEquals(INITIAL_KEYS + 1 + LOOKAHEAD_SIZE, group.numKeys());
+        assertEquals(INITIAL_KEYS + 1 + LOOKAHEAD_SIZE + group.getLookaheadThreshold(), group.numKeys());
         assertTrue(a1.isP2SHAddress());
 
         Address a2 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         assertEquals(a1, a2);
-        assertEquals(INITIAL_KEYS + 1 + LOOKAHEAD_SIZE, group.numKeys());
+        assertEquals(INITIAL_KEYS + 1 + LOOKAHEAD_SIZE + group.getLookaheadThreshold(), group.numKeys());
 
         Address a3 = group.currentAddress(KeyChain.KeyPurpose.CHANGE);
         assertNotEquals(a2, a3);
@@ -170,7 +170,8 @@ public class KeyChainGroupTest {
         Address a2 = group.freshAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         assertTrue(a1.isP2SHAddress());
         assertNotEquals(a1, a2);
-        assertEquals(INITIAL_KEYS + 2 + LOOKAHEAD_SIZE, group.numKeys());
+        // numKeys does not include following chains. Possibly it should.
+        assertEquals(INITIAL_KEYS + 1 + group.getLookaheadSize() + group.getLookaheadThreshold(), group.numKeys());
 
         Address a3 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         assertEquals(a2, a3);
@@ -280,14 +281,14 @@ public class KeyChainGroupTest {
         assertEquals(INITIAL_KEYS * 2, group.getBloomFilterElementCount());
         ECKey key1 = group.freshKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         ECKey key2 = new ECKey();
-        final int size = (INITIAL_KEYS + LOOKAHEAD_SIZE + 1 /* for the just created key */) * 2;
+        final int size = (INITIAL_KEYS + LOOKAHEAD_SIZE + group.getLookaheadThreshold() + 1 /* for the just created key */) * 2;
         assertEquals(size, group.getBloomFilterElementCount());
         BloomFilter filter = group.getBloomFilter(size, 0.001, (long)(Math.random() * Long.MAX_VALUE));
         assertTrue(filter.contains(key1.getPubKeyHash()));
         assertTrue(filter.contains(key1.getPubKey()));
         assertFalse(filter.contains(key2.getPubKey()));
-        // Check that the filter contains the lookahead buffer.
+        // Check that the filter contains the lookahead buffer and threshold zone.
-        for (int i = 0; i < LOOKAHEAD_SIZE; i++) {
+        for (int i = 0; i < LOOKAHEAD_SIZE + group.getLookaheadThreshold(); i++) {
             ECKey k = group.freshKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
             assertTrue(filter.contains(k.getPubKeyHash()));
         }
@@ -307,8 +308,8 @@ public class KeyChainGroupTest {
         assertTrue(group.findRedeemScriptFromPubHash(address.getHash160()) != null);
         KeyChainGroup group2 = createMarriedKeyChainGroup();
         group2.freshAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
-        // test address from lookahead zone
+        // test address from lookahead zone and lookahead threshold zone
-        for (int i = 0; i < LOOKAHEAD_SIZE; i++) {
+        for (int i = 0; i < LOOKAHEAD_SIZE + group.getLookaheadThreshold(); i++) {
             address = group.freshAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
             assertTrue(group2.findRedeemScriptFromPubHash(address.getHash160()) != null);
         }
@@ -321,7 +322,7 @@ public class KeyChainGroupTest {
         // only leaf keys are used for populating bloom filter, so initial number is zero
         assertEquals(0, group.getBloomFilterElementCount());
         Address address1 = group.freshAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
-        final int size = (LOOKAHEAD_SIZE + 1 /* for the just created key */) * 2;
+        final int size = (LOOKAHEAD_SIZE + group.getLookaheadThreshold() + 1 /* for the just created key */) * 2;
         assertEquals(size, group.getBloomFilterElementCount());
         BloomFilter filter = group.getBloomFilter(size, 0.001, (long)(Math.random() * Long.MAX_VALUE));
         assertTrue(filter.contains(address1.getHash160()));
@@ -330,7 +331,7 @@ public class KeyChainGroupTest {
         assertFalse(filter.contains(address2.getHash160()));
 
         // Check that the filter contains the lookahead buffer.
-        for (int i = 0; i < LOOKAHEAD_SIZE; i++) {
+        for (int i = 0; i < LOOKAHEAD_SIZE + group.getLookaheadThreshold(); i++) {
             Address address = group.freshAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
             assertTrue(filter.contains(address.getHash160()));
         }
@@ -421,9 +422,9 @@ public class KeyChainGroupTest {
         group.freshKey(KeyChain.KeyPurpose.RECEIVE_FUNDS);
         group.freshKey(KeyChain.KeyPurpose.CHANGE);
         List<Protos.Key> protoKeys1 = group.serializeToProtobuf();
-        assertEquals(3 + (LOOKAHEAD_SIZE + 1) * 2, protoKeys1.size());
+        assertEquals(3 + (LOOKAHEAD_SIZE + group.getLookaheadThreshold() + 1) * 2, protoKeys1.size());
         group = KeyChainGroup.fromProtobufUnencrypted(params, protoKeys1);
-        assertEquals(3 + (LOOKAHEAD_SIZE + 1) * 2, group.serializeToProtobuf().size());
+        assertEquals(3 + (LOOKAHEAD_SIZE + group.getLookaheadThreshold() + 1) * 2, group.serializeToProtobuf().size());
     }
 
     @Test
@@ -553,4 +554,14 @@ public class KeyChainGroupTest {
         byte[] truncatedBytes = Arrays.copyOfRange(key.getSecretBytes(), 0, 16);
         assertArrayEquals(entropy, truncatedBytes);
     }
+
+    @Test
+    public void markAsUsed() throws Exception {
+        Address addr1 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
+        Address addr2 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
+        assertEquals(addr1, addr2);
+        group.markPubKeyHashAsUsed(addr1.getHash160());
+        Address addr3 = group.currentAddress(KeyChain.KeyPurpose.RECEIVE_FUNDS);
+        assertNotEquals(addr2, addr3);
+    }
 }

core/src/test/resources/com/google/bitcoin/wallet/deterministic-wallet-serialization.txt

@@ -148,6 +148,24 @@ deterministic_key {
   path: 11
 }
 
+type: DETERMINISTIC_KEY
+public_key: "\003\000\334\035\2400n\26636x\316\327\3666\271\375K\031\366\307\221J@\331@dL\232Bv\324\262"
+deterministic_key {
+  chain_code: "\207^n\317\370\t\207\341*\\\360\026iBRTQ#\252Z\237\373{\315\333\004\340nA9\252\352"
+  path: 2147483648
+  path: 0
+  path: 12
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\003?\v\222\341\321\"@\2624\336H\221\2570bC\251\377\000vm\032\357U\250Dl:\200\020Q-"
+deterministic_key {
+  chain_code: "\205R\372F\245\232*\0351Y??\321\362I\222ne\277H+\304\rL\234\313\016|\a\372a\a"
+  path: 2147483648
+  path: 0
+  path: 13
+}
+
 type: DETERMINISTIC_KEY
 public_key: "\002\225b\3515\202\233\335\320.7\265\274uh\230N\242\254\317J\364\331\2345\220)\362\334\216\202\\"
 deterministic_key {
@@ -246,3 +264,30 @@ deterministic_key {
   path: 1
   path: 10
 }
+
+type: DETERMINISTIC_KEY
+public_key: "\003-\221uJ\237\240\320\025\031w\001V\276\030j\217Z\222 \330\253\332\330F\216\377D\311\211\277\351\230"
+deterministic_key {
+  chain_code: "\241\363\245\033W\f*J\026\021\210Ic\2318a\"\036\302\005+\220\003\3364\211o\362\225R~\340"
+  path: 2147483648
+  path: 1
+  path: 11
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\002V\3212\255\n\367\226%]0\342\003\317\031\350\265K\247\035\005}\004[N\262\262\376Ed\261j\377"
+deterministic_key {
+  chain_code: "0\236\330H\354\237\016\367-/E\344\311\024\353\307\331\367n\017\250n\351\000\204\233\224\242L\343&;"
+  path: 2147483648
+  path: 1
+  path: 12
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\003\301\302\254\214C}\362f\315GV\033]\257\a\231\t[\001=\0046\213\220\341S\324\266\202&\206N"
+deterministic_key {
+  chain_code: "\272\\\225\354.UQ8\264\346\a\310h\350\031\227\024c\340\337;W7\f\322\301\304\232P\360\373\035"
+  path: 2147483648
+  path: 1
+  path: 13
+}

core/src/test/resources/com/google/bitcoin/wallet/watching-wallet-serialization.txt

@@ -134,6 +134,24 @@ deterministic_key {
   path: 11
 }
 
+type: DETERMINISTIC_KEY
+public_key: "\003\000\334\035\2400n\26636x\316\327\3666\271\375K\031\366\307\221J@\331@dL\232Bv\324\262"
+deterministic_key {
+  chain_code: "\207^n\317\370\t\207\341*\\\360\026iBRTQ#\252Z\237\373{\315\333\004\340nA9\252\352"
+  path: 2147483648
+  path: 0
+  path: 12
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\003?\v\222\341\321\"@\2624\336H\221\2570bC\251\377\000vm\032\357U\250Dl:\200\020Q-"
+deterministic_key {
+  chain_code: "\205R\372F\245\232*\0351Y??\321\362I\222ne\277H+\304\rL\234\313\016|\a\372a\a"
+  path: 2147483648
+  path: 0
+  path: 13
+}
+
 type: DETERMINISTIC_KEY
 public_key: "\002\225b\3515\202\233\335\320.7\265\274uh\230N\242\254\317J\364\331\2345\220)\362\334\216\202\\"
 deterministic_key {
@@ -232,3 +250,30 @@ deterministic_key {
   path: 1
   path: 10
 }
+
+type: DETERMINISTIC_KEY
+public_key: "\003-\221uJ\237\240\320\025\031w\001V\276\030j\217Z\222 \330\253\332\330F\216\377D\311\211\277\351\230"
+deterministic_key {
+  chain_code: "\241\363\245\033W\f*J\026\021\210Ic\2318a\"\036\302\005+\220\003\3364\211o\362\225R~\340"
+  path: 2147483648
+  path: 1
+  path: 11
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\002V\3212\255\n\367\226%]0\342\003\317\031\350\265K\247\035\005}\004[N\262\262\376Ed\261j\377"
+deterministic_key {
+  chain_code: "0\236\330H\354\237\016\367-/E\344\311\024\353\307\331\367n\017\250n\351\000\204\233\224\242L\343&;"
+  path: 2147483648
+  path: 1
+  path: 12
+}
+
+type: DETERMINISTIC_KEY
+public_key: "\003\301\302\254\214C}\362f\315GV\033]\257\a\231\t[\001=\0046\213\220\341S\324\266\202&\206N"
+deterministic_key {
+  chain_code: "\272\\\225\354.UQ8\264\346\a\310h\350\031\227\024c\340\337;W7\f\322\301\304\232P\360\373\035"
+  path: 2147483648
+  path: 1
+  path: 13
+}