First part of block chain handling rework.

- Store the block chain using a get/put interface keyed by hash, 
  so we can add disk storage later.
  
- Add unit tests for difficulty transitions. Move some stuff into 
  NetworkParameters to make that easier.
  
- Track the best chain using total work done. Inform the wallet
  when a re-org takes place. Wallet currently doesn't do anything
  with this beyond informing the event listeners.

With this patch we're getting closer to a correct SPV implementation.

src/com/google/bitcoin/core/Block.java

@@ -39,6 +39,9 @@ public class Block extends Message {
     /** A value for difficultyTarget (nBits) that allows half of all possible hash solutions. Used in unit testing. */
     static final long EASIEST_DIFFICULTY_TARGET = 0x207fFFFFL;
 
+    // For unit testing. If not zero, use this instead of the current time.
+    static long fakeClock = 0;
+
     private long version;
     private byte[] prevBlockHash;
     private byte[] merkleRoot;
@@ -157,6 +160,18 @@ public class Block extends Message {
         return LARGEST_HASH.divide(target.add(BigInteger.ONE));
     }
 
+    /** Returns a copy of the block, but without any transactions. */
+    public Block cloneAsHeader() {
+        try {
+            Block block = new Block(params, bitcoinSerialize());
+            block.transactions = null;
+            return block;
+        } catch (ProtocolException e) {
+            // Should not be able to happen unless our state is internally inconsistent.
+            throw new RuntimeException(e);
+        }
+    }
+
     /**
      * Returns a multi-line string containing a description of the contents of the block. Use for debugging purposes
      * only.
@@ -206,7 +221,7 @@ public class Block extends Message {
     public BigInteger getDifficultyTargetAsInteger() throws VerificationException {
         BigInteger target = Utils.decodeCompactBits(difficultyTarget);
         if (target.compareTo(BigInteger.valueOf(0)) <= 0 || target.compareTo(params.proofOfWorkLimit) > 0)
-            throw new VerificationException("Difficulty target is bad");
+            throw new VerificationException("Difficulty target is bad: " + target.toString());
         return target;
     }
 
@@ -235,7 +250,9 @@ public class Block extends Message {
     }
     
     private void checkTimestamp() throws VerificationException {
-        if (time > (System.currentTimeMillis() / 1000) + ALLOWED_TIME_DRIFT)
+        // Allow injection of a fake clock to allow unit testing.
+        long currentTime = fakeClock != 0 ? fakeClock : System.currentTimeMillis() / 1000;
+        if (time > currentTime + ALLOWED_TIME_DRIFT)
             throw new VerificationException("Block too far in future");
     }
     

src/com/google/bitcoin/core/BlockChain.java

@@ -19,7 +19,6 @@ package com.google.bitcoin.core;
 import java.math.BigInteger;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.LinkedList;
 
 import static com.google.bitcoin.core.Utils.LOG;
 
@@ -48,19 +47,40 @@ import static com.google.bitcoin.core.Utils.LOG;
  * or if we connect to a peer that doesn't send us blocks in order.
  */
 public class BlockChain {
-    // This is going away.
+    /** Keeps a map of block hashes to StoredBlocks. */
-    private final LinkedList<Block> blockChain;
+    protected BlockStore blockStore;
 
-    private final NetworkParameters params;
+    /**
-    private final Wallet wallet;
+     * Tracks the top of the best known chain.<p>
+     *
+     * Following this one down to the genesis block produces the story of the economy from the creation of BitCoin
+     * until the present day. The chain head can change if a new set of blocks is received that results in a chain of
+     * greater work than the one obtained by following this one down. In that case a reorganize is triggered,
+     * potentially invalidating transactions in our wallet.
+     */
+    protected StoredBlock chainHead;
+
+    protected final NetworkParameters params;
+    protected final Wallet wallet;
 
     // Holds blocks that we have received but can't plug into the chain yet, eg because they were created whilst we
     // were downloading the block chain.
     private final ArrayList<Block> unconnectedBlocks = new ArrayList<Block>();
 
     public BlockChain(NetworkParameters params, Wallet wallet) {
-        blockChain = new LinkedList<Block>();
+        // TODO: Let the user pass in a BlockStore object so they can choose how to store the headers.
-        blockChain.add(params.genesisBlock);
+        blockStore = new MemoryBlockStore();
+        try {
+            // Set up the genesis block. When we start out fresh, it is by definition the top of the chain.
+            Block genesisHeader = params.genesisBlock.cloneAsHeader();
+            chainHead = new StoredBlock(genesisHeader, genesisHeader.getWork(), 0);
+            blockStore.put(chainHead);
+        } catch (BlockStoreException e) {
+            // Cannot happen.
+        } catch (VerificationException e) {
+            // Genesis block always verifies.
+        }
+
         this.params = params;
         this.wallet = wallet;
     }
@@ -71,11 +91,18 @@ public class BlockChain {
      * If the block can be connected to the chain, returns true.
      */
     public synchronized boolean add(Block block) throws VerificationException, ScriptException {
-        return add(block, true);
+        try {
+            return add(block, true);
+        } catch (BlockStoreException e) {
+            // TODO: Figure out a better way to propagate this exception to the user.
+            throw new RuntimeException(e);
+        }
     }
 
-    private synchronized boolean add(Block block, boolean tryConnecting) throws VerificationException, ScriptException {
+    private synchronized boolean add(Block block, boolean tryConnecting)
+            throws BlockStoreException, VerificationException, ScriptException {
         try {
+            // Prove the block is internally valid: hash is lower than target, merkle root is correct and so on.
             block.verify();
         } catch (VerificationException e) {
             LOG("Failed to verify block: " + e.toString());
@@ -98,37 +125,64 @@ public class BlockChain {
         }
         // We don't need the transaction data anymore. Free up some memory.
         block.transactions = null;
-        // We know prev is OK because it's in the blockMap, that means we accepted it.
+
-        Block prev = blockChain.getLast();
+        if (blockStore.get(block.getHash()) != null) {
-        if (prev.equals(block)) {
+            LOG("Already have block");
-            LOG("Re-received block that is currently on top of the chain.");
             return true;
         }
-        if (!Arrays.equals(block.getPrevBlockHash(), prev.getHash())) {
+        StoredBlock storedPrev = blockStore.get(block.getPrevBlockHash());
-            // The block does not fit onto the top of the chain. It can either be:
+        if (storedPrev == null) {
-            //   - Entirely unconnected. This can happen when a new block is solved and broadcast whilst we are in
+            // We can't find the previous block. Probably we are still in the process of downloading the chain and a
-            //     the process of downloading the block chain.
+            // block was solved whilst we were doing it. We put it to one side and try to connect it later when we
-            //   - Connected to an earlier block in the chain than the top one. This can happen when there is a
+            // have more blocks.
-            //     split in the chain.
+            LOG("Block does not connect: " + block.getHashAsString());
-            //   - Connected as part of an orphan chain, ie a chain of blocks that does not connect to the genesis
-            //     block.
-            // TODO: We don't support most of these cases today and it's a high priority to do so.
             unconnectedBlocks.add(block);
             return false;
+        } else {
+            // The block connects to somewhere on the chain. Not necessarily the top of the best known chain.
+            checkDifficultyTransitions(storedPrev, block);
+            StoredBlock newStoredBlock = buildStoredBlock(storedPrev, block);
+            // Store it.
+            blockStore.put(newStoredBlock);
+            // TODO: Break the assumption of object equality here.
+            if (storedPrev == chainHead) {
+                // This block connects to the best known block, it is a normal continuation of the system.
+                chainHead = newStoredBlock;
+                LOG("Received new block, chain is now " + chainHead.height + " blocks high");
+            } else {
+                // This block connects to somewhere other than the top of the chain.
+                if (newStoredBlock.moreWorkThan(chainHead)) {
+                    // This chain has overtaken the one we currently believe is best. Reorganize is required.
+                    wallet.reorganize(chainHead, newStoredBlock);
+                    // Update the pointer to the best known block.
+                    chainHead = newStoredBlock;
+                } else {
+                    LOG("Received a block which forks the chain, but it did not cause a reorganize.");
+                }
+            }
         }
-        checkDifficultyTransitions(prev, block);
+
-        // The block is OK so let's build the rest of the chain on it.
-        block.prevBlock = prev;
-        blockChain.add(block);
         if (tryConnecting)
             tryConnectingUnconnected();
+
         return true;
     }
 
+    /**
+     * Calculates the additional fields a StoredBlock holds given the previous block in the chain and the new block.
+     */
+    private StoredBlock buildStoredBlock(StoredBlock storedPrev, Block block) throws VerificationException {
+        // Stored blocks track total work done in this chain, because the canonical chain is the one that represents
+        // the largest amount of work done not the tallest.
+        BigInteger chainWork = storedPrev.chainWork.add(block.getWork());
+        int height = storedPrev.height + 1;
+        return new StoredBlock(block, chainWork, height);
+    }
+
     /**
      * For each block in unconnectedBlocks, see if we can now fit it on top of the chain and if so, do so.
      */
-    private void tryConnectingUnconnected() throws VerificationException, ScriptException {
+    private void tryConnectingUnconnected() throws VerificationException, ScriptException, BlockStoreException {
         // For each block in our unconnected list, try and fit it onto the head of the chain. If we succeed remove it
         // from the list and keep going. If we changed the head of the list at the end of the round,
         // try again until we can't fit anything else on the top.
@@ -137,13 +191,18 @@ public class BlockChain {
             blocksConnectedThisRound = 0;
             for (int i = 0; i < unconnectedBlocks.size(); i++) {
                 Block block = unconnectedBlocks.get(i);
-                if (Arrays.equals(block.getPrevBlockHash(), blockChain.getLast().getHash())) {
+                // Look up the blocks previous.
-                    // False here ensures we don't recurse infinitely downwards when connecting huge chains.
+                StoredBlock prev = blockStore.get(block.getPrevBlockHash());
-                    add(block, false);
+                if (prev == null) {
-                    unconnectedBlocks.remove(i);
+                    // This is still an unconnected/orphan block.
-                    i--;  // The next iteration of the for loop will make "i" point to the right index again.
+                    continue;
-                    blocksConnectedThisRound++;
                 }
+                // Otherwise we can connect it now.
+                // False here ensures we don't recurse infinitely downwards when connecting huge chains.
+                add(block, false);
+                unconnectedBlocks.remove(i);
+                i--;  // The next iteration of the for loop will make "i" point to the right index again.
+                blocksConnectedThisRound++;
             }
             if (blocksConnectedThisRound > 0) {
                 LOG("Connected " + blocksConnectedThisRound + " floating blocks.");
@@ -151,34 +210,48 @@ public class BlockChain {
         } while (blocksConnectedThisRound > 0);
     }
 
-    static private final int TARGET_TIMESPAN = 14 * 24 * 60 * 60;
+    /**
-    static private final int TARGET_SPACING = 10 * 60;
+     * Throws an exception if the blocks difficulty is not correct.
-    static private final int INTERVAL = TARGET_TIMESPAN / TARGET_SPACING;
+     */
-
+    private void checkDifficultyTransitions(StoredBlock storedPrev, Block next)
-    private void checkDifficultyTransitions(Block prev, Block next) throws VerificationException {
+            throws BlockStoreException, VerificationException {
+        Block prev = storedPrev.header;
         // Is this supposed to be a difficulty transition point?
-        if (blockChain.size() % INTERVAL != 0) {
+        if ((storedPrev.height + 1) % params.interval != 0) {
             // No ... so check the difficulty didn't actually change.
             if (next.getDifficultyTarget() != prev.getDifficultyTarget())
-                throw new VerificationException("Unexpected change in difficulty at height " + blockChain.size() +
+                throw new VerificationException("Unexpected change in difficulty at height " + storedPrev.height +
                         ": " + Long.toHexString(next.getDifficultyTarget()) + " vs " +
                         Long.toHexString(prev.getDifficultyTarget()));
             return;
         }
 
-        Block blockIntervalAgo = blockChain.get(blockChain.size() - INTERVAL);
+        // We need to find a block far back in the chain. It's OK that this is expensive because it only occurs every
+        // two weeks after the initial block chain download.
+        StoredBlock cursor = blockStore.get(prev.getHash());
+        for (int i = 0; i < params.interval - 1; i++) {
+            if (cursor == null) {
+                // This should never happen. If it does, it means we are following an incorrect or busted chain.
+                throw new VerificationException(
+                        "Difficulty transition point but we did not find a way back to the genesis block.");
+            }
+            cursor = blockStore.get(cursor.header.getPrevBlockHash());
+        }
+
+        Block blockIntervalAgo = cursor.header;
         int timespan = (int) (prev.getTime() - blockIntervalAgo.getTime());
         // Limit the adjustment step.
-        if (timespan < TARGET_TIMESPAN / 4)
+        if (timespan < params.targetTimespan / 4)
-            timespan = TARGET_TIMESPAN / 4;
+            timespan = params.targetTimespan / 4;
-        if (timespan > TARGET_TIMESPAN * 4)
+        if (timespan > params.targetTimespan * 4)
-            timespan = TARGET_TIMESPAN * 4;
+            timespan = params.targetTimespan * 4;
 
         BigInteger newDifficulty = Utils.decodeCompactBits(blockIntervalAgo.getDifficultyTarget());
         newDifficulty = newDifficulty.multiply(BigInteger.valueOf(timespan));
-        newDifficulty = newDifficulty.divide(BigInteger.valueOf(TARGET_TIMESPAN));
+        newDifficulty = newDifficulty.divide(BigInteger.valueOf(params.targetTimespan));
 
         if (newDifficulty.compareTo(params.proofOfWorkLimit) > 0) {
+            LOG("Difficulty hit proof of work limit: " + newDifficulty.toString(16));
             newDifficulty = params.proofOfWorkLimit;
         }
 
@@ -190,7 +263,7 @@ public class BlockChain {
         newDifficulty = newDifficulty.and(mask);
 
         if (newDifficulty.compareTo(receivedDifficulty) != 0)
-            throw new VerificationException("Calculated difficulty bits do not match what network provided: " +
+            throw new VerificationException("Network provided difficulty bits do not match what was calculated: " +
                     receivedDifficulty.toString(16) + " vs " + newDifficulty.toString(16));
     }
 
@@ -232,10 +305,11 @@ public class BlockChain {
     }
 
     /**
-     * Returns the highest known block or null if the chain is empty (top block is genesis).
+     * Returns the block at the head of the current best chain. This is the block which represents the greatest
+     * amount of cumulative work done.
      */
-    public synchronized Block getTopBlock() {
+    public synchronized StoredBlock getChainHead() {
-        return blockChain.getLast();
+        return chainHead;
     }
 
 

src/com/google/bitcoin/core/BlockStore.java

@@ -0,0 +1,42 @@
+/**
+ * Copyright 2011 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.bitcoin.core;
+
+/**
+ * An implementor of BlockStore saves StoredBlock objects to disk. Different implementations store them in
+ * different ways. An in-memory implementation (MemoryBlockStore) exists for unit testing but real apps will want to
+ * use implementations that save to disk.<p>
+ *
+ * A BlockStore is a map of hashes to StoredBlock. The hash is the double digest of the BitCoin serialization
+ * of the block header, <b>not</b> the header with the extra data as well.<p>
+ *
+ * BlockStores are thread safe.
+ */
+interface BlockStore {
+    /**
+     * Saves the given block header+extra data. The key isn't specified explicitly as it can be calculated from the
+     * StoredBlock directly. Can throw if there is a problem with the underlying storage layer such as running out of
+     * disk space.
+     */
+    void put(StoredBlock block) throws BlockStoreException;
+
+    /**
+     * Returns the StoredBlock given a hash. The returned values block.getHash() method will be equal to the
+     * parameter. If no such block is found, returns null.
+     */
+    StoredBlock get(byte[] hash) throws BlockStoreException;
+}

src/com/google/bitcoin/core/BlockStoreException.java

@@ -0,0 +1,23 @@
+/**
+ * Copyright 2011 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.bitcoin.core;
+
+/**
+ * Thrown when something goes wrong with storing a block. Examples: out of disk space.
+ */
+public class BlockStoreException extends Exception {
+}

src/com/google/bitcoin/core/MemoryBlockStore.java

@@ -0,0 +1,44 @@
+/**
+ * Copyright 2011 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.bitcoin.core;
+
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Keeps {@link StoredBlock}s in memory. Used primarily for unit testing.
+ */
+class MemoryBlockStore implements BlockStore {
+    // We use a ByteBuffer to hold hashes here because the Java array equals()/hashcode() methods do not operate on
+    // the contents of the array but just inherit the default Object behavior. ByteBuffer provides the functionality
+    // needed to act as a key in a map.
+    private Map<ByteBuffer, StoredBlock> blockMap;
+
+    MemoryBlockStore() {
+        blockMap = new HashMap<ByteBuffer, StoredBlock>();
+    }
+
+    public synchronized void put(StoredBlock block) throws BlockStoreException {
+        byte[] hash = block.header.getHash();
+        blockMap.put(ByteBuffer.wrap(hash), block);
+    }
+
+    public synchronized StoredBlock get(byte[] hash) throws BlockStoreException {
+        return blockMap.get(ByteBuffer.wrap(hash));
+    }
+}

src/com/google/bitcoin/core/NetworkParameters.java

@@ -48,6 +48,15 @@ public class NetworkParameters implements Serializable {
     public long packetMagic;
     /** First byte of a base58 encoded address. */
     public byte addressHeader;
+    /** How many blocks pass between difficulty adjustment periods. BitCoin standardises this to be 2015. */
+    public int interval;
+    /**
+     * How much time in seconds is supposed to pass between "interval" blocks. If the actual elapsed time is
+     * significantly different from this value, the network difficulty formula will produce a different value. Both
+     * test and production BitCoin networks use 2 weeks (1209600 seconds).
+     */
+    public int targetTimespan;
+
 
     // The genesis block is the first block in the chain and is a shared, well known block of data containin a
     // headline from the Times, as well as initialization values for that chain. The testnet uses a similar genesis
@@ -74,13 +83,20 @@ public class NetworkParameters implements Serializable {
         return genesisBlock;
     }
 
+    static private final int TARGET_TIMESPAN = 14 * 24 * 60 * 60;  // 2 weeks per difficulty cycle, on average.
+    static private final int TARGET_SPACING = 10 * 60;  // 10 minutes per block.
+    static private final int INTERVAL = TARGET_TIMESPAN / TARGET_SPACING;
+
     /** Sets up the given NetworkParameters with testnet values. */
     private static NetworkParameters createTestNet(NetworkParameters n) {
         // Genesis hash is 0000000224b1593e3ff16a0e3b61285bbc393a39f78c8aa48c456142671f7110
+        // The proof of work limit has to start with 00, as otherwise the value will be interpreted as negative.
         n.proofOfWorkLimit = new BigInteger("0000000fffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16);
         n.packetMagic = 0xfabfb5daL;
         n.port = 18333;
         n.addressHeader = 111;
+        n.interval = INTERVAL;
+        n.targetTimespan = TARGET_TIMESPAN;
         n.genesisBlock = createGenesis(n);
         n.genesisBlock.setTime(1296688602L);
         n.genesisBlock.setDifficultyTarget(0x1d07fff8L);
@@ -103,6 +119,8 @@ public class NetworkParameters implements Serializable {
         n.port = 8333;
         n.packetMagic = 0xf9beb4d9L;
         n.addressHeader = 0;
+        n.interval = INTERVAL;
+        n.targetTimespan = TARGET_TIMESPAN;
         n.genesisBlock = createGenesis(n);
         n.genesisBlock.setDifficultyTarget(0x1d00ffffL);
         n.genesisBlock.setTime(1231006505L);
@@ -118,6 +136,8 @@ public class NetworkParameters implements Serializable {
         n = createTestNet(n);
         n.proofOfWorkLimit = new BigInteger("00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16);
         n.genesisBlock.setDifficultyTarget(Block.EASIEST_DIFFICULTY_TARGET);
+        n.interval = 10;
+        n.targetTimespan = 200000000;  // 6 years. Just a very big number.
         return n;
     }
 }

src/com/google/bitcoin/core/Peer.java

@@ -17,7 +17,6 @@
 package com.google.bitcoin.core;
 
 import java.io.IOException;
-import java.net.SocketException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.LinkedList;
@@ -145,9 +144,11 @@ public class Peer {
             }
         } catch (VerificationException e) {
             // We don't want verification failures to kill the thread.
-            LOG("Ignoring verification exception.");
+            LOG(e.toString());
+            e.printStackTrace();
         } catch (ScriptException e) {
             // We don't want script failures to kill the thread.
+            LOG(e.toString());
             e.printStackTrace();
         }
     }
@@ -296,16 +297,16 @@ public class Peer {
         //
         // So this is a complicated process but it has the advantage that we can download a chain of enormous length
         // in a relatively stateless manner and with constant/bounded memory usage.
+        LOG("Peer.blockChainDownload: " + Utils.bytesToHexString(toHash));
 
         // TODO: Block locators should be abstracted out rather than special cased here.
         List<byte[]> blockLocator = new LinkedList<byte[]>();
         // We don't do the exponential thinning here, so if we get onto a fork of the chain we will end up
         // redownloading the whole thing again.
         blockLocator.add(params.genesisBlock.getHash());
-        Block topBlock = blockChain.getTopBlock();
+        Block topBlock = blockChain.getChainHead().header;
-        if (topBlock != null) {
+        if (!topBlock.equals(params.genesisBlock))
             blockLocator.add(0, topBlock.getHash());
-        }
         GetBlocksMessage message = new GetBlocksMessage(params, blockLocator, toHash);
         conn.writeMessage(NetworkConnection.MSG_GETBLOCKS, message);
     }

src/com/google/bitcoin/core/StoredBlock.java

@@ -0,0 +1,59 @@
+/**
+ * Copyright 2011 Google Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.google.bitcoin.core;
+
+import java.math.BigInteger;
+
+/**
+ * Wraps a {@link Block} object with extra data that can be derived from the block chain but is slow or inconvenient to
+ * calculate. By storing it alongside the block header we reduce the amount of work required significantly.
+ * Recalculation is slow because the fields are cumulative - to find the chainWork you have to iterate over every
+ * block in the chain back to the genesis block, which involves lots of seeking/loading etc. So we just keep a
+ * running total: it's a disk space vs cpu/io tradeoff.<p>
+ *
+ * StoredBlocks are put inside a {@link BlockStore} which saves them to memory or disk.
+ */
+class StoredBlock {
+    /**
+     * The block header this object wraps. The referenced block object must not have any transactions in it.
+     */
+    Block header;
+
+    /**
+     * The total sum of work done in this block, and all the blocks below it in the chain. Work is a measure of how
+     * many tries are needed to solve a block. If the target is set to cover 10% of the total hash value space,
+     * then the work represented by a block is 10.
+     */
+    BigInteger chainWork;
+
+    /**
+     * Position in the chain for this block. The genesis block has a height of zero.
+     */
+    int height;
+
+    StoredBlock(Block header, BigInteger chainWork, int height) {
+        assert header.transactions == null : "Should not have transactions in a block header object";
+        this.header = header;
+        this.chainWork = chainWork;
+        this.height = height;
+    }
+
+    /** Returns true if this objects chainWork is higher than the others. */
+    boolean moreWorkThan(StoredBlock other) {
+        return chainWork.compareTo(other.chainWork) > 0;
+    }
+}

src/com/google/bitcoin/core/Wallet.java

@@ -23,6 +23,8 @@ import java.util.Arrays;
 import java.util.LinkedList;
 import java.util.List;
 
+import static com.google.bitcoin.core.Utils.LOG;
+
 /**
  * A Wallet stores keys and a record of transactions that have not yet been spent. Thus, it is capable of
  * providing transactions on demand that meet a given combined value. Once a transaction
@@ -133,7 +135,7 @@ public class Wallet implements Serializable {
                 }
                 TransactionOutput linkedOutput = t.outputs.get((int) input.outpoint.index);
                 assert !linkedOutput.isSpent : "Double spend was accepted by network?";
-                Utils.LOG("Saw a record of me spending " + Utils.bitcoinValueToFriendlyString(linkedOutput.getValue())
+                LOG("Saw a record of me spending " + Utils.bitcoinValueToFriendlyString(linkedOutput.getValue())
                         + " BTC");
                 linkedOutput.isSpent = true;
                 // Are all the outputs on this TX that are mine now spent? Note that some of the outputs may not
@@ -156,9 +158,9 @@ public class Wallet implements Serializable {
                 }
             }
         }
-        Utils.LOG("Received " + Utils.bitcoinValueToFriendlyString(tx.getValueSentToMe(this)));
+        LOG("Received " + Utils.bitcoinValueToFriendlyString(tx.getValueSentToMe(this)));
         unspent.add(tx);
-        Utils.LOG("Balance is now: " + Utils.bitcoinValueToFriendlyString(getBalance()));
+        LOG("Balance is now: " + Utils.bitcoinValueToFriendlyString(getBalance()));
 
         // Inform anyone interested that we have new coins. Note: we may be re-entered by the event listener,
         // so we must not make assumptions about our state after this loop returns! For example,
@@ -265,7 +267,7 @@ public class Wallet implements Serializable {
      * @return a new {@link Transaction} or null if we cannot afford this send.
      */
     synchronized Transaction createSend(Address address,  BigInteger nanocoins, Address changeAddress) {
-        Utils.LOG("Creating send tx to " + address.toString() + " for " +
+        LOG("Creating send tx to " + address.toString() + " for " +
                 Utils.bitcoinValueToFriendlyString(nanocoins));
         // To send money to somebody else, we need to do the following:
         //  - Gather up transactions with unspent outputs until we have sufficient value.
@@ -283,7 +285,7 @@ public class Wallet implements Serializable {
         }
         // Can we afford this?
         if (valueGathered.compareTo(nanocoins) < 0) {
-            Utils.LOG("Insufficient value in wallet for send, missing " +
+            LOG("Insufficient value in wallet for send, missing " +
                     Utils.bitcoinValueToFriendlyString(nanocoins.subtract(valueGathered)));
             // TODO: Should throw an exception here.
             return null;
@@ -295,7 +297,7 @@ public class Wallet implements Serializable {
             // The value of the inputs is greater than what we want to send. Just like in real life then,
             // we need to take back some coins ... this is called "change". Add another output that sends the change
             // back to us.
-            Utils.LOG("  with " + Utils.bitcoinValueToFriendlyString(change) + " coins change");
+            LOG("  with " + Utils.bitcoinValueToFriendlyString(change) + " coins change");
             sendTx.addOutput(new TransactionOutput(params, change, changeAddress));
         }
         for (TransactionOutput output : gathered) {
@@ -369,4 +371,36 @@ public class Wallet implements Serializable {
         }
         return builder.toString();
     }
+
+    /**
+     * Called by the {@link BlockChain} when the best chain (representing total work done) has changed. In this case,
+     * we need to go through our transactions and find out if any have become invalid. It's possible for our balance
+     * to go down in this case: money we thought we had can suddenly vanish if the rest of the network agrees it
+     * should be so.
+     */
+    void reorganize(StoredBlock chainHead, StoredBlock newStoredBlock) {
+        // This runs on any peer thread with the block chain synchronized. Thus we do not have to worry about it
+        // being called simultaneously or repeatedly.
+        LOG("Re-organize!");
+        LOG("Old chain head: " + chainHead.header.toString());
+        LOG("New chain head: " + newStoredBlock.header.toString());
+
+        // TODO: Implement me!
+        // For each transaction we have to track which blocks they appeared in. Once a re-org takes place,
+        // we will have to find all transactions in the old branch, all transactions in the new branch and find the
+        // difference of those sets. If there is no difference it means we the user doesn't really care about this
+        // re-org but we still need to update the transaction block pointers.
+        boolean affectedUs = true;
+
+        // We should only trigger this event if the re-org actually impacted our wallet. Otherwise the user is
+        // unlikely to care.
+        if (affectedUs) {
+            // Inform event listeners that a re-org took place.
+            for (WalletEventListener l : eventListeners) {
+                synchronized (l) {
+                    l.onReorganize();
+                }
+            }
+        }
+    }
 }

src/com/google/bitcoin/core/WalletEventListener.java

@@ -3,19 +3,37 @@ package com.google.bitcoin.core;
 import java.math.BigInteger;
 
 /**
- * Implementing WalletEventListener allows you to learn when a wallets balance has changed.
+ * Implementing a subclass WalletEventListener allows you to learn when the contents of the wallet changes due to
+ * receiving money or a block chain re-organize. Methods are called with the event listener object locked so your
+ * implementation does not have to be thread safe. The default method implementations do nothing.
  */
-public interface WalletEventListener {
+public abstract class WalletEventListener {
     /**
      * This is called on a Peer thread when a block is received that sends some coins to you. Note that this will
-     * also be called when downloading the block chain as the wallet balance catches up,
+     * also be called when downloading the block chain as the wallet balance catches up so if you don't want that
-     * so if you don't want that register the event listener after the chain is downloaded. It's safe to use methods
+     * register the event listener after the chain is downloaded. It's safe to use methods of wallet during the
-     * of wallet during the execution of this callback.
+     * execution of this callback.
      *
      * @param wallet The wallet object that received the coins/
      * @param tx The transaction which sent us the coins.
      * @param prevBalance Balance before the coins were received.
      * @param newBalance Current balance of the wallet.
      */
-    public void onCoinsReceived(Wallet wallet, Transaction tx, BigInteger prevBalance, BigInteger newBalance);
+    public void onCoinsReceived(Wallet wallet, Transaction tx, BigInteger prevBalance, BigInteger newBalance) {
+    }
+
+    /**
+     * This is called on a Peer thread when a block is received that triggers a block chain re-organization.<p>
+     *
+     * A re-organize means that the consensus (chain) of the network has diverged and now changed from what we
+     * believed it was previously. Usually this won't matter because the new consensus will include all our old
+     * transactions assuming we are playing by the rules. However it's theoretically possible for our balance to
+     * change in arbitrary ways, most likely, we could lose some money we thought we had.<p>
+     *
+     * It is safe to use methods of wallet whilst inside this callback.
+     *
+     * TODO: Finish this interface.
+     */
+    public void onReorganize() {
+    }
 }

tests/com/google/bitcoin/core/BlockChainTest.java

@@ -18,28 +18,35 @@ package com.google.bitcoin.core;
 
 import com.google.bitcoin.bouncycastle.util.encoders.Hex;
 import org.junit.Before;
-import org.junit.Ignore;
 import org.junit.Test;
 
 import java.math.BigInteger;
 
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
 
 // Tests still to write:
 //   - Rest of checkDifficultyTransitions: verify we don't accept invalid transitions.
 //   - Fragmented chains can be joined together.
-//   - Longest chain is selected based on total difficulty not length.
+//   - Longest testNetChain is selected based on total difficulty not length.
 //   - Many more ...
 public class BlockChainTest {
-    private static final NetworkParameters params = NetworkParameters.testNet();
+    private static final NetworkParameters testNet = NetworkParameters.testNet();
+    private BlockChain testNetChain;
+
+    private Wallet wallet;
     private BlockChain chain;
+    private Address coinbaseTo;
+    private NetworkParameters unitTestParams;
 
     @Before
     public void setUp() {
-        Wallet wallet = new Wallet(params);
+        testNetChain = new BlockChain(testNet, new Wallet(testNet));
-        chain = new BlockChain(params, wallet);
+
+        unitTestParams = NetworkParameters.unitTests();
+        wallet = new Wallet(unitTestParams);
+        wallet.addKey(new ECKey());
+        coinbaseTo = wallet.keychain.get(0).toAddress(unitTestParams);
+        chain = new BlockChain(unitTestParams, wallet);
     }
 
     @Test
@@ -47,7 +54,7 @@ public class BlockChainTest {
         // Check that we can plug a few blocks together.
         // Block 1 from the testnet.
         Block b1 = getBlock1();
-        assertTrue(chain.add(b1));
+        assertTrue(testNetChain.add(b1));
         // Block 2 from the testnet.
         Block b2 = getBlock2();
 
@@ -55,37 +62,60 @@ public class BlockChainTest {
         long n = b2.getNonce();
         try {
             b2.setNonce(12345);
-            chain.add(b2);
+            testNetChain.add(b2);
             fail();
         } catch (VerificationException e) {
             b2.setNonce(n);
         }
-        assertTrue(chain.add(b2));
+        // Now it works because we reset the nonce.
+        assertTrue(testNetChain.add(b2));
     }
 
     private Block createNextBlock(Address to, Block prev) throws VerificationException {
+        return createNextBlock(to, prev, Block.EASIEST_DIFFICULTY_TARGET, System.currentTimeMillis() / 1000);
+    }
+
+    private Block createNextBlock(Address to, Block prev, long difficultyTarget,
+                                  long time) throws VerificationException {
         Block b = new Block(prev.params);
-        b.setDifficultyTarget(Block.EASIEST_DIFFICULTY_TARGET);
+        b.setDifficultyTarget(difficultyTarget);
         b.addCoinbaseTransaction(to);
         b.setPrevBlockHash(prev.getHash());
+        b.setTime(time);
         b.solve();
         b.verify();
         return b;
     }
 
-    @Test @Ignore
+    @Test
-    public void testForking() throws Exception {
+    public void testUnconnectedBlocks() throws Exception {
-        // Check that if the block chain forks, we end up using the right one.
-        NetworkParameters unitTestParams = NetworkParameters.unitTests();
-        Wallet wallet = new Wallet(unitTestParams);
-        wallet.addKey(new ECKey());
-        Address coinbaseTo = wallet.keychain.get(0).toAddress(unitTestParams);
-        // Start by building a couple of blocks on top of the genesis block.
         Block b1 = createNextBlock(coinbaseTo, unitTestParams.genesisBlock);
         Block b2 = createNextBlock(coinbaseTo, b1);
-        chain = new BlockChain(unitTestParams, wallet);
+        Block b3 = createNextBlock(coinbaseTo, b2);
-        chain.add(b1);
+        // Connected.
-        chain.add(b2);
+        assertTrue(chain.add(b1));
+        // Unconnected.
+        assertFalse(chain.add(b3));
+    }
+
+    @Test
+    public void testForking() throws Exception {
+        // Check that if the block chain forks, we end up using the right one.
+        // Start by building a couple of blocks on top of the genesis block.
+        final boolean[] flags = new boolean[1];
+        flags[0] = false;
+        wallet.addEventListener(new WalletEventListener() {
+            @Override
+            public void onReorganize() {
+                flags[0] = true;
+            }
+        });
+
+        Block b1 = createNextBlock(coinbaseTo, unitTestParams.genesisBlock);
+        Block b2 = createNextBlock(coinbaseTo, b1);
+        assertTrue(chain.add(b1));
+        assertTrue(chain.add(b2));
+        assertFalse(flags[0]);
         // We got two blocks which generated 50 coins each, to us.
         assertEquals("100.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
         // We now have the following chain:
@@ -99,33 +129,65 @@ public class BlockChainTest {
         // Nothing should happen at this point. We saw b2 first so it takes priority.
         Address someOtherGuy = new ECKey().toAddress(unitTestParams);
         Block b3 = createNextBlock(someOtherGuy, b1);
-        chain.add(b3);
+        assertTrue(chain.add(b3));
+        assertFalse(flags[0]);  // No re-org took place.
         assertEquals("100.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
         // Now we add another block to make the alternative chain longer.
-        chain.add(createNextBlock(someOtherGuy, b3));
+        assertTrue(chain.add(createNextBlock(someOtherGuy, b3)));
+        assertTrue(flags[0]);  // Re-org took place.
+        flags[0] = false;
         //
         //     genesis -> b1 -> b2
         //                  \-> b3 -> b4
         //
         // We lost some coins! b2 is no longer a part of the best chain so our balance should drop to 50 again.
-        assertEquals("50.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
+        if (false) {
-        // ... and back to the first chain
+            // These tests do not pass currently, as wallet handling of re-orgs isn't implemented.
-        Block b5 = createNextBlock(coinbaseTo, b2);
+            assertEquals("50.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
-        Block b6 = createNextBlock(coinbaseTo, b5);
+            // ... and back to the first testNetChain
-        chain.add(b5);
+            Block b5 = createNextBlock(coinbaseTo, b2);
-        chain.add(b6);
+            Block b6 = createNextBlock(coinbaseTo, b5);
-        //
+            assertTrue(chain.add(b5));
-        //     genesis -> b1 -> b2 -> b5 -> b6
+            assertTrue(chain.add(b6));
-        //                  \-> b3 -> b4
+            //
-        //
+            //     genesis -> b1 -> b2 -> b5 -> b6
-        assertEquals("200.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
+            //                  \-> b3 -> b4
+            //
+            assertEquals("200.00", Utils.bitcoinValueToFriendlyString(wallet.getBalance()));
+        }
+    }
+
+    @Test
+    public void testDifficultyTransitions() throws Exception {
+        // Add a bunch of blocks in a loop until we reach a difficulty transition point. The unit test params have an
+        // artificially shortened period.
+        Block prev = unitTestParams.genesisBlock;
+        Block.fakeClock = System.currentTimeMillis() / 1000;
+        for (int i = 0; i < unitTestParams.interval - 1; i++) {
+            Block newBlock = createNextBlock(coinbaseTo, prev, Block.EASIEST_DIFFICULTY_TARGET, Block.fakeClock);
+            assertTrue(chain.add(newBlock));
+            prev = newBlock;
+            // The fake chain should seem to be "fast" for the purposes of difficulty calculations.
+            Block.fakeClock += 2;
+        }
+        // Now add another block that has no difficulty adjustment, it should be rejected.
+        try {
+            chain.add(createNextBlock(coinbaseTo, prev));
+            fail();
+        } catch (VerificationException e) {
+        }
+        // Create a new block with the right difficulty target given our blistering speed relative to the huge amount
+        // of time it's supposed to take (set in the unit test network parameters).
+        Block b = createNextBlock(coinbaseTo, prev, 0x201fFFFFL, Block.fakeClock);
+        assertTrue(chain.add(b));
+        // Successfully traversed a difficulty transition period.
     }
 
     @Test
     public void testBadDifficulty() throws Exception {
-        assertTrue(chain.add(getBlock1()));
+        assertTrue(testNetChain.add(getBlock1()));
         Block b2 = getBlock2();
-        assertTrue(chain.add(b2));
+        assertTrue(testNetChain.add(b2));
         NetworkParameters params2 = NetworkParameters.testNet();
         Block bad = new Block(params2);
         // Merkle root can be anything here, doesn't matter.
@@ -139,7 +201,7 @@ public class BlockChainTest {
         // solutions.
         bad.setDifficultyTarget(Block.EASIEST_DIFFICULTY_TARGET);
         try {
-            chain.add(bad);
+            testNetChain.add(bad);
             // The difficulty target above should be rejected on the grounds of being easier than the networks
             // allowable difficulty.
             fail();
@@ -151,7 +213,7 @@ public class BlockChainTest {
         params2.proofOfWorkLimit = new BigInteger
                 ("00ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", 16);
         try {
-            chain.add(bad);
+            testNetChain.add(bad);
             // We should not get here as the difficulty target should not be changing at this point.
             fail();
         } catch (VerificationException e) {
@@ -161,8 +223,9 @@ public class BlockChainTest {
         // TODO: Test difficulty change is not out of range when a transition period becomes valid.
     }
 
+    // Some blocks from the test net.
     private Block getBlock2() throws Exception {
-        Block b2 = new Block(params);
+        Block b2 = new Block(testNet);
         b2.setMerkleRoot(Hex.decode("addc858a17e21e68350f968ccd384d6439b64aafa6c193c8b9dd66320470838b"));
         b2.setNonce(2642058077L);
         b2.setTime(1296734343L);
@@ -173,7 +236,7 @@ public class BlockChainTest {
     }
 
     private Block getBlock1() throws Exception {
-        Block b1 = new Block(params);
+        Block b1 = new Block(testNet);
         b1.setMerkleRoot(Hex.decode("0e8e58ecdacaa7b3c6304a35ae4ffff964816d2b80b62b58558866ce4e648c10"));
         b1.setNonce(236038445);
         b1.setTime(1296734340);