From 891cdcc02d3d767a0889bb3c438f3aad175d5a5a Mon Sep 17 00:00:00 2001 From: Miron Cuperman Date: Tue, 17 Jan 2012 11:06:01 -0800 Subject: [PATCH 1/2] Remove dependence of WalletProtobufSerializer on BlockStore, migrate to Transaction.appearsInHashes --- src/com/google/bitcoin/core/Transaction.java | 83 ++++++++++--------- src/com/google/bitcoin/core/Wallet.java | 18 ++-- .../store/WalletProtobufSerializer.java | 43 +++++----- tests/com/google/bitcoin/core/WalletTest.java | 20 ++++- .../store/WalletProtobufSerializerTest.java | 26 +++--- 5 files changed, 105 insertions(+), 85 deletions(-) diff --git a/src/com/google/bitcoin/core/Transaction.java b/src/com/google/bitcoin/core/Transaction.java index 781a2406..050a551d 100644 --- a/src/com/google/bitcoin/core/Transaction.java +++ b/src/com/google/bitcoin/core/Transaction.java @@ -22,7 +22,7 @@ import org.slf4j.LoggerFactory; import java.io.*; import java.math.BigInteger; import java.util.*; - + import static com.google.bitcoin.core.Utils.*; /** @@ -56,15 +56,9 @@ public class Transaction extends ChildMessage implements Serializable { private long lockTime; - // This is only stored in Java serialization. It records which blocks (and their height + work) the transaction - // has been included in. For most transactions this set will have a single member. In the case of a chain split a - // transaction may appear in multiple blocks but only one of them is part of the best chain. It's not valid to - // have an identical transaction appear in two blocks in the same chain but this invariant is expensive to check, - // so it's not directly enforced anywhere. - // - // If this transaction is not stored in the wallet, appearsIn is null. + // This is being migrated to appearsInHashes Set appearsIn; - + // Stored only in Java serialization. This is either the time the transaction was broadcast as measured from the // local clock, or the time from the block in which it was included. Note that this can be changed by re-orgs so // the wallet may update this field. Old serialized transactions don't have this field, thus null is valid. @@ -78,6 +72,15 @@ public class Transaction extends ChildMessage implements Serializable { // Data about how confirmed this tx is. Serialized, may be null. private TransactionConfidence confidence; + // This records which blocks the transaction + // has been included in. For most transactions this set will have a single member. In the case of a chain split a + // transaction may appear in multiple blocks but only one of them is part of the best chain. It's not valid to + // have an identical transaction appear in two blocks in the same chain but this invariant is expensive to check, + // so it's not directly enforced anywhere. + // + // If this transaction is not stored in the wallet, appearsInHashes is null. + Set appearsInHashes; + public Transaction(NetworkParameters params) { super(params); version = 1; @@ -190,8 +193,21 @@ public class Transaction extends ChildMessage implements Serializable { * Returns a set of blocks which contain the transaction, or null if this transaction doesn't have that data * because it's not stored in the wallet or because it has never appeared in a block. */ - public Set getAppearsIn() { - return appearsIn; + public Collection getAppearsInHashes() { + if (appearsInHashes != null) + return appearsInHashes; + + if (appearsIn != null) { + assert appearsInHashes == null; + log.info("Migrating a tx to appearsInHashes"); + appearsInHashes = new HashSet(appearsIn.size()); + for (StoredBlock block : appearsIn) { + appearsInHashes.add(block.getHeader().getHash()); + } + appearsIn = null; + } + + return appearsInHashes; } /** @@ -207,31 +223,36 @@ public class Transaction extends ChildMessage implements Serializable { * used by the wallet to ensure transactions that appear on side chains are recorded properly even though the * block stores do not save the transaction data at all.

* - * If there is a re-org this will be called once for each block that was previously seen, to update which block + *

If there is a re-org this will be called once for each block that was previously seen, to update which block * is the best chain. The best chain block is guaranteed to be called last. So this must be idempotent. * + *

Sets updatedAt to be the earliest valid block time where this tx was seen + * * @param block The {@link StoredBlock} in which the transaction has appeared. * @param bestChain whether to set the updatedAt timestamp from the block header (only if not already set) */ public void setBlockAppearance(StoredBlock block, boolean bestChain) { - if (bestChain && updatedAt == null) { - updatedAt = new Date(block.getHeader().getTimeSeconds() * 1000); + long blockTime = block.getHeader().getTimeSeconds() * 1000; + if (bestChain && (updatedAt == null || updatedAt.getTime() == 0 || updatedAt.getTime() > blockTime)) { + updatedAt = new Date(blockTime); } - if (appearsIn == null) { - appearsIn = new HashSet(); - } - appearsIn.add(block); + + addBlockAppearance(block.getHeader().getHash()); if (bestChain) { - if (updatedAt == null) { - updatedAt = new Date(block.getHeader().getTimeSeconds() * 1000); - } // This can cause event listeners on TransactionConfidence to run. After this line completes, the wallets // state may have changed! getConfidence().setAppearedAtChainHeight(block.getHeight()); } } + public void addBlockAppearance(final Sha256Hash blockHash) { + if (appearsInHashes == null) { + appearsInHashes = new HashSet(); + } + appearsInHashes.add(blockHash); + } + /** Called by the wallet once a re-org means we don't appear in the best chain anymore. */ void notifyNotOnBestChain() { getConfidence().setConfidenceType(TransactionConfidence.ConfidenceType.NOT_IN_BEST_CHAIN); @@ -324,26 +345,12 @@ public class Transaction extends ChildMessage implements Serializable { /** * Returns the earliest time at which the transaction was seen (broadcast or included into the chain), - * or null if that information isn't available. + * or the epoch if that information isn't available. */ public Date getUpdateTime() { if (updatedAt == null) { - // Older wallets did not store this field. If we can, fill it out based on the block pointers. We might - // "guess wrong" in the case of transactions appearing on chain forks, but this is unlikely to matter in - // practice. Note, some patched copies of BitCoinJ store dates in this field that do not correspond to any - // block but rather broadcast time. - if (appearsIn == null || appearsIn.size() == 0) { - // Transaction came from somewhere that doesn't provide time info. - return null; - } - long earliestTimeSecs = Long.MAX_VALUE; - // We might return a time that is different to the best chain, as we don't know here which block is part - // of the active chain and which are simply inactive. We just ignore this for now. - // TODO: At some point we'll want to store storing full block headers in the wallet. Remove at that time. - for (StoredBlock b : appearsIn) { - earliestTimeSecs = Math.min(b.getHeader().getTimeSeconds(), earliestTimeSecs); - } - updatedAt = new Date(earliestTimeSecs * 1000); + // Older wallets did not store this field. Set to the epoch. + updatedAt = new Date(0); } return updatedAt; } diff --git a/src/com/google/bitcoin/core/Wallet.java b/src/com/google/bitcoin/core/Wallet.java index c9d511f6..8556e433 100644 --- a/src/com/google/bitcoin/core/Wallet.java +++ b/src/com/google/bitcoin/core/Wallet.java @@ -1158,10 +1158,18 @@ public class Wallet implements Serializable { // // receive() has been called on the block that is triggering the re-org before this is called. + List oldBlockHashes = new ArrayList(oldBlocks.size()); + List newBlockHashes = new ArrayList(newBlocks.size()); log.info("Old part of chain (top to bottom):"); - for (StoredBlock b : oldBlocks) log.info(" {}", b.getHeader().getHashAsString()); + for (StoredBlock b : oldBlocks) { + log.info(" {}", b.getHeader().getHashAsString()); + oldBlockHashes.add(b.getHeader().getHash()); + } log.info("New part of chain (top to bottom):"); - for (StoredBlock b : newBlocks) log.info(" {}", b.getHeader().getHashAsString()); + for (StoredBlock b : newBlocks) { + log.info(" {}", b.getHeader().getHashAsString()); + newBlockHashes.add(b.getHeader().getHash()); + } // Transactions that appear in the old chain segment. Map oldChainTransactions = new HashMap(); @@ -1177,12 +1185,12 @@ public class Wallet implements Serializable { all.putAll(spent); all.putAll(inactive); for (Transaction tx : all.values()) { - Set appearsIn = tx.getAppearsIn(); + Collection appearsIn = tx.getAppearsInHashes(); assert appearsIn != null; // If the set of blocks this transaction appears in is disjoint with one of the chain segments it means // the transaction was never incorporated by a miner into that side of the chain. - boolean inOldSection = !Collections.disjoint(appearsIn, oldBlocks); - boolean inNewSection = !Collections.disjoint(appearsIn, newBlocks); + boolean inOldSection = !Collections.disjoint(appearsIn, oldBlockHashes); + boolean inNewSection = !Collections.disjoint(appearsIn, newBlockHashes); boolean inCommonSection = !inNewSection && !inOldSection; if (inCommonSection) { diff --git a/src/com/google/bitcoin/store/WalletProtobufSerializer.java b/src/com/google/bitcoin/store/WalletProtobufSerializer.java index 35c569ef..45cf29e5 100644 --- a/src/com/google/bitcoin/store/WalletProtobufSerializer.java +++ b/src/com/google/bitcoin/store/WalletProtobufSerializer.java @@ -16,21 +16,9 @@ package com.google.bitcoin.store; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.math.BigInteger; -import java.util.Date; -import java.util.HashMap; -import java.util.Map; - -import org.bitcoinj.wallet.Protos; - -import com.google.bitcoin.core.AddressFormatException; import com.google.bitcoin.core.ECKey; import com.google.bitcoin.core.NetworkParameters; import com.google.bitcoin.core.Sha256Hash; -import com.google.bitcoin.core.StoredBlock; import com.google.bitcoin.core.Transaction; import com.google.bitcoin.core.TransactionInput; import com.google.bitcoin.core.TransactionOutPoint; @@ -40,6 +28,16 @@ import com.google.bitcoin.core.WalletTransaction; import com.google.protobuf.ByteString; import com.google.protobuf.TextFormat; +import org.bitcoinj.wallet.Protos; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.math.BigInteger; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; + /** * Serialize and de-serialize a wallet to a protobuf stream. * @@ -59,7 +57,7 @@ public class WalletProtobufSerializer { walletProto.writeTo(output); } - public static String walletToText(Wallet wallet) throws IOException { + public static String walletToText(Wallet wallet) { Protos.Wallet walletProto = walletToProto(wallet); return TextFormat.printToString(walletProto); @@ -128,23 +126,23 @@ public class WalletProtobufSerializer { if (spentBy != null) { outputBuilder .setSpentByTransactionHash(ByteString.copyFrom(spentBy.getParentTransaction().getHash().getBytes())) - .setSpentByTransactionIndex((int)spentBy.getParentTransaction().getInputs().indexOf(spentBy)); // FIXME + .setSpentByTransactionIndex(spentBy.getParentTransaction().getInputs().indexOf(spentBy)); } txBuilder.addTransactionOutput(outputBuilder); } // Handle which blocks tx was seen in - if (tx.getAppearsIn() != null) { - for (StoredBlock block : tx.getAppearsIn()) { - txBuilder.addBlockHash(ByteString.copyFrom(block.getHeader().getHash().getBytes())); + if (tx.getAppearsInHashes() != null) { + for (Sha256Hash hash : tx.getAppearsInHashes()) { + txBuilder.addBlockHash(ByteString.copyFrom(hash.getBytes())); } } return txBuilder.build(); } - public static Wallet readWallet(InputStream input, NetworkParameters params, BlockStore store) - throws IOException, AddressFormatException, BlockStoreException { + public static Wallet readWallet(InputStream input, NetworkParameters params) + throws IOException { WalletProtobufSerializer serializer = new WalletProtobufSerializer(); Protos.Wallet walletProto = Protos.Wallet.parseFrom(input); if (!params.getId().equals(walletProto.getNetworkIdentifier())) @@ -164,7 +162,7 @@ public class WalletProtobufSerializer { // Read all transactions and create outputs for (Protos.Transaction txProto : walletProto.getTransactionList()) { - serializer.readTransaction(txProto, params, store); + serializer.readTransaction(txProto, params); } // Create transactions inputs pointing to transactions @@ -188,8 +186,7 @@ public class WalletProtobufSerializer { } - private void readTransaction(Protos.Transaction txProto, - NetworkParameters params, BlockStore store) throws BlockStoreException { + private void readTransaction(Protos.Transaction txProto, NetworkParameters params) { Transaction tx = new Transaction(params, txProto.getVersion(), new Sha256Hash(txProto.getHash().toByteArray())); if (txProto.hasUpdatedAt()) tx.setUpdateTime(new Date(txProto.getUpdatedAt())); @@ -207,7 +204,7 @@ public class WalletProtobufSerializer { } for (ByteString blockHash : txProto.getBlockHashList()) { - tx.setBlockAppearance(store.get(new Sha256Hash(blockHash.toByteArray())), false); + tx.addBlockAppearance(new Sha256Hash(blockHash.toByteArray())); } if (txProto.hasLockTime()) { diff --git a/tests/com/google/bitcoin/core/WalletTest.java b/tests/com/google/bitcoin/core/WalletTest.java index b25f2bf9..a8cefe6b 100644 --- a/tests/com/google/bitcoin/core/WalletTest.java +++ b/tests/com/google/bitcoin/core/WalletTest.java @@ -23,6 +23,7 @@ import org.junit.Before; import org.junit.Test; import java.math.BigInteger; +import java.util.HashSet; import java.util.List; import static com.google.bitcoin.core.TestUtils.createFakeBlock; @@ -460,12 +461,10 @@ public class WalletTest { // Verify we can handle the case of older wallets in which the timestamp is null (guessed from the // block appearances list). tx1.updatedAt = null; - tx2.updatedAt = null; + tx3.updatedAt = null; // Check we got them back in order. transactions = wallet.getTransactionsByTime(); - assertEquals(tx3, transactions.get(0)); - assertEquals(tx2, transactions.get(1)); - assertEquals(tx1, transactions.get(2)); + assertEquals(tx2, transactions.get(0)); assertEquals(3, transactions.size()); } @@ -482,5 +481,18 @@ public class WalletTest { wallet.addKey(new ECKey()); assertEquals(now + 60, wallet.getEarliestKeyCreationTime()); } + + @Test + public void transactionAppearsInMigration() throws Exception { + // Test migration from appearsIn to appearsInHashes + Transaction tx1 = createFakeTx(params, Utils.toNanoCoins(1, 0), myAddress); + StoredBlock b1 = createFakeBlock(params, blockStore, tx1).storedBlock; + tx1.appearsIn = new HashSet(); + tx1.appearsIn.add(b1); + assertEquals(1, tx1.getAppearsInHashes().size()); + assertTrue(tx1.getAppearsInHashes().contains(b1.getHeader().getHash())); + assertNull(tx1.appearsIn); + } + // Support for offline spending is tested in PeerGroupTest } diff --git a/tests/com/google/bitcoin/store/WalletProtobufSerializerTest.java b/tests/com/google/bitcoin/store/WalletProtobufSerializerTest.java index d3f1ebd6..ecc78e59 100644 --- a/tests/com/google/bitcoin/store/WalletProtobufSerializerTest.java +++ b/tests/com/google/bitcoin/store/WalletProtobufSerializerTest.java @@ -3,18 +3,9 @@ package com.google.bitcoin.store; import static com.google.bitcoin.core.TestUtils.createFakeTx; import static com.google.bitcoin.core.Utils.toNanoCoins; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.math.BigInteger; - -import org.bitcoinj.wallet.Protos; -import org.junit.Before; -import org.junit.Test; +import static org.junit.Assert.*; import com.google.bitcoin.core.Address; -import com.google.bitcoin.core.AddressFormatException; import com.google.bitcoin.core.BlockChain; import com.google.bitcoin.core.ECKey; import com.google.bitcoin.core.NetworkParameters; @@ -22,14 +13,20 @@ import com.google.bitcoin.core.Transaction; import com.google.bitcoin.core.Utils; import com.google.bitcoin.core.Wallet; -import static org.junit.Assert.*; +import org.bitcoinj.wallet.Protos; +import org.junit.Before; +import org.junit.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.math.BigInteger; public class WalletProtobufSerializerTest { static final NetworkParameters params = NetworkParameters.unitTests(); private ECKey myKey; private Address myAddress; private Wallet wallet; - private MemoryBlockStore blockStore; @Before public void setUp() throws Exception { @@ -37,7 +34,6 @@ public class WalletProtobufSerializerTest { myAddress = myKey.toAddress(params); wallet = new Wallet(params); wallet.addKey(myKey); - blockStore = new MemoryBlockStore(params); } @Test @@ -101,11 +97,11 @@ public class WalletProtobufSerializerTest { } } - private Wallet roundTrip(Wallet wallet) throws IOException, AddressFormatException, BlockStoreException { + private Wallet roundTrip(Wallet wallet) throws IOException { ByteArrayOutputStream output = new ByteArrayOutputStream(); //System.out.println(WalletProtobufSerializer.walletToText(wallet)); WalletProtobufSerializer.writeWallet(wallet, output); ByteArrayInputStream input = new ByteArrayInputStream(output.toByteArray()); - return WalletProtobufSerializer.readWallet(input, params, blockStore); + return WalletProtobufSerializer.readWallet(input, params); } } From ed5adf3ea8597a80a88fecf40cdf2943ddf36d78 Mon Sep 17 00:00:00 2001 From: Miron Cuperman Date: Thu, 19 Jan 2012 08:41:41 -0800 Subject: [PATCH 2/2] Disable the gen source directory for now as it is unused --- pom.xml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pom.xml b/pom.xml index 87ff29e7..8b44f370 100644 --- a/pom.xml +++ b/pom.xml @@ -254,6 +254,7 @@ +