3
0
mirror of https://github.com/Qortal/altcoinj.git synced 2025-01-31 15:22:16 +00:00

Implement mempool-test support in BitcoindComparisonTool

This commit is contained in:
Matt Corallo 2013-07-23 18:23:28 +02:00 committed by Mike Hearn
parent 60220aa62d
commit a7ec6a1db3
3 changed files with 229 additions and 81 deletions

View File

@ -22,12 +22,14 @@ import com.google.bitcoin.store.FullPrunedBlockStore;
import com.google.bitcoin.store.H2FullPrunedBlockStore;
import com.google.bitcoin.utils.BlockFileLoader;
import com.google.bitcoin.utils.BriefLogFormatter;
import com.google.bitcoin.utils.Threading;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.net.InetAddress;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
/**
* A tool for comparing the blocks which are accepted/rejected by bitcoind/bitcoinj
@ -43,7 +45,8 @@ public class BitcoindComparisonTool {
private static PeerGroup peers;
private static Sha256Hash bitcoindChainHead;
private static volatile Peer bitcoind;
private static volatile InventoryMessage mostRecentInv = null;
public static void main(String[] args) throws Exception {
BriefLogFormatter.init();
System.out.println("USAGE: bitcoinjBlockStoreLocation runLargeReorgs(1/0) [port=18444]");
@ -55,7 +58,7 @@ public class BitcoindComparisonTool {
blockFile.deleteOnExit();
FullBlockTestGenerator generator = new FullBlockTestGenerator(params);
BlockAndValidityList blockList = generator.getBlocksToTest(true, runLargeReorgs, blockFile);
RuleList blockList = generator.getBlocksToTest(false, runLargeReorgs, blockFile);
Iterator<Block> blocks = new BlockFileLoader(params, Arrays.asList(blockFile));
try {
@ -75,6 +78,7 @@ public class BitcoindComparisonTool {
peers.addAddress(new PeerAddress(InetAddress.getByName("localhost"), args.length > 2 ? Integer.parseInt(args[2]) : params.getPort()));
final Set<Sha256Hash> blocksRequested = Collections.synchronizedSet(new HashSet<Sha256Hash>());
final AtomicInteger unexpectedInvs = new AtomicInteger(0);
peers.addEventListener(new AbstractPeerEventListener() {
@Override
public void onPeerConnected(Peer peer, int peerCount) {
@ -104,9 +108,30 @@ public class BitcoindComparisonTool {
if (item.type == InventoryItem.Type.Block)
blocksRequested.add(item.hash);
return null;
} else if (m instanceof InventoryMessage) {
if (mostRecentInv != null) {
log.error("Got an inv when we weren't expecting one");
unexpectedInvs.incrementAndGet();
}
mostRecentInv = (InventoryMessage) m;
}
return m;
}
}, Threading.SAME_THREAD);
peers.addPeerFilterProvider(new PeerFilterProvider() {
@Override public long getEarliestKeyCreationTime() {
return Long.MAX_VALUE;
}
@Override public int getBloomFilterElementCount() {
return 1;
}
@Override public BloomFilter getBloomFilter(int size, double falsePositiveRate, long nTweak) {
BloomFilter filter = new BloomFilter(1, 0.99, 0);
filter.setMatchAll();
return filter;
}
});
bitcoindChainHead = params.getGenesisBlock().getHash();
@ -125,61 +150,97 @@ public class BitcoindComparisonTool {
int differingBlocks = 0;
int invalidBlocks = 0;
for (BlockAndValidity block : blockList.list) {
boolean threw = false;
Block nextBlock = blocks.next();
try {
if (chain.add(nextBlock) != block.connects) {
log.error("Block didn't match connects flag on block \"" + block.blockName + "\"");
int mempoolRulesFailed = 0;
for (Rule rule : blockList.list) {
if (rule instanceof BlockAndValidity) {
BlockAndValidity block = (BlockAndValidity) rule;
boolean threw = false;
Block nextBlock = blocks.next();
try {
if (chain.add(nextBlock) != block.connects) {
log.error("Block didn't match connects flag on block \"" + block.ruleName + "\"");
invalidBlocks++;
}
} catch (VerificationException e) {
threw = true;
if (!block.throwsException) {
log.error("Block didn't match throws flag on block \"" + block.ruleName + "\"");
e.printStackTrace();
invalidBlocks++;
} else if (block.connects) {
log.error("Block didn't match connects flag on block \"" + block.ruleName + "\"");
e.printStackTrace();
invalidBlocks++;
}
}
if (!threw && block.throwsException) {
log.error("Block didn't match throws flag on block \"" + block.ruleName + "\"");
invalidBlocks++;
} else if (!chain.getChainHead().getHeader().getHash().equals(block.hashChainTipAfterBlock)) {
log.error("New block head didn't match the correct value after block \"" + block.ruleName + "\"");
invalidBlocks++;
} else if (chain.getChainHead().getHeight() != block.heightAfterBlock) {
log.error("New block head didn't match the correct height after block " + block.ruleName);
invalidBlocks++;
}
} catch (VerificationException e) {
threw = true;
if (!block.throwsException) {
log.error("Block didn't match throws flag on block \"" + block.blockName + "\"");
e.printStackTrace();
invalidBlocks++;
} else if (block.connects) {
log.error("Block didn't match connects flag on block \"" + block.blockName + "\"");
e.printStackTrace();
invalidBlocks++;
InventoryMessage message = new InventoryMessage(params);
message.addBlock(nextBlock);
bitcoind.sendMessage(message);
// bitcoind doesn't request blocks inline so we can't rely on a ping for synchronization
for (int i = 0; !blocksRequested.contains(nextBlock.getHash()); i++) {
if (i % 20 == 19)
log.error("bitcoind still hasn't requested block " + block.ruleName + " with hash " + nextBlock.getHash());
Thread.sleep(50);
}
bitcoind.sendMessage(nextBlock);
locator.clear();
locator.add(bitcoindChainHead);
bitcoind.sendMessage(new GetHeadersMessage(params, locator, hashTo));
bitcoind.ping().get();
if (!chain.getChainHead().getHeader().getHash().equals(bitcoindChainHead)) {
differingBlocks++;
log.error("bitcoind and bitcoinj acceptance differs on block \"" + block.ruleName + "\"");
}
log.info("Block \"" + block.ruleName + "\" completed processing");
} else if (rule instanceof MemoryPoolState) {
MemoryPoolMessage message = new MemoryPoolMessage();
bitcoind.sendMessage(message);
bitcoind.ping().get();
if (mostRecentInv == null && !((MemoryPoolState) rule).mempool.isEmpty()) {
log.error("bitcoind had an empty mempool, but we expected some transactions on rule " + rule.ruleName);
mempoolRulesFailed++;
} else if (mostRecentInv != null && ((MemoryPoolState) rule).mempool.isEmpty()) {
log.error("bitcoind had a non-empty mempool, but we expected an empty one on rule " + rule.ruleName);
mempoolRulesFailed++;
} else if (mostRecentInv != null) {
Set<InventoryItem> originalRuleSet = new HashSet<InventoryItem>(((MemoryPoolState)rule).mempool);
boolean matches = mostRecentInv.items.size() == ((MemoryPoolState)rule).mempool.size();
for (InventoryItem item : mostRecentInv.items)
if (!((MemoryPoolState) rule).mempool.remove(item))
matches = false;
if (matches)
continue;
log.error("bitcoind's mempool didn't match what we were expecting on rule " + rule.ruleName);
log.info(" bitcoind's mempool was: ");
for (InventoryItem item : mostRecentInv.items)
log.info(" " + item.hash);
log.info(" The expected mempool was: ");
for (InventoryItem item : originalRuleSet)
log.info(" " + item.hash);
mempoolRulesFailed++;
}
mostRecentInv = null;
} else {
log.error("Unknown rule");
}
if (!threw && block.throwsException) {
log.error("Block didn't match throws flag on block \"" + block.blockName + "\"");
invalidBlocks++;
} else if (!chain.getChainHead().getHeader().getHash().equals(block.hashChainTipAfterBlock)) {
log.error("New block head didn't match the correct value after block \"" + block.blockName + "\"");
invalidBlocks++;
} else if (chain.getChainHead().getHeight() != block.heightAfterBlock) {
log.error("New block head didn't match the correct height after block " + block.blockName);
invalidBlocks++;
}
InventoryMessage message = new InventoryMessage(params);
message.addBlock(nextBlock);
bitcoind.sendMessage(message);
// bitcoind doesn't request blocks inline so we can't rely on a ping for synchronization
for (int i = 0; !blocksRequested.contains(nextBlock.getHash()); i++) {
if (i % 20 == 19)
log.error("bitcoind still hasn't requested block " + block.blockName);
Thread.sleep(50);
}
bitcoind.sendMessage(nextBlock);
locator.clear();
locator.add(bitcoindChainHead);
bitcoind.sendMessage(new GetHeadersMessage(params, locator, hashTo));
bitcoind.ping().get();
if (!chain.getChainHead().getHeader().getHash().equals(bitcoindChainHead)) {
differingBlocks++;
log.error("bitcoind and bitcoinj acceptance differs on block \"" + block.blockName + "\"");
}
log.info("Block \"" + block.blockName + "\" completed processing");
}
log.info("Done testing.\n" +
"Blocks which were not handled the same between bitcoind/bitcoinj: " + differingBlocks + "\n" +
"Blocks which should/should not have been accepted but weren't/were: " + invalidBlocks);
System.exit(differingBlocks > 0 || invalidBlocks > 0 ? 1 : 0);
"Blocks which should/should not have been accepted but weren't/were: " + invalidBlocks + "\n" +
"Transactions which were/weren't in memory pool but shouldn't/should have been: " + mempoolRulesFailed + "\n" +
"Unexpected inv messages: " + unexpectedInvs.get());
System.exit(differingBlocks > 0 || invalidBlocks > 0 || mempoolRulesFailed > 0 || unexpectedInvs.get() > 0 ? 1 : 0);
}
}

View File

@ -14,15 +14,19 @@ import java.util.*;
import static com.google.bitcoin.script.ScriptOpCodes.*;
class BlockAndValidity {
/**
* Represents a block which is sent to the tested application and which the application must either reject or accept,
* depending on the flags in the rule
*/
class BlockAndValidity extends Rule {
Block block;
boolean connects;
boolean throwsException;
Sha256Hash hashChainTipAfterBlock;
int heightAfterBlock;
String blockName;
public BlockAndValidity(Map<Sha256Hash, Integer> blockToHeightMap, Block block, boolean connects, boolean throwsException, Sha256Hash hashChainTipAfterBlock, int heightAfterBlock, String blockName) {
super(blockName);
if (connects && throwsException)
throw new RuntimeException("A block cannot connect if an exception was thrown while adding it.");
this.block = block;
@ -30,8 +34,7 @@ class BlockAndValidity {
this.throwsException = throwsException;
this.hashChainTipAfterBlock = hashChainTipAfterBlock;
this.heightAfterBlock = heightAfterBlock;
this.blockName = blockName;
// Double-check that we are always marking any given block at the same height
Integer height = blockToHeightMap.get(hashChainTipAfterBlock);
if (height != null)
@ -41,6 +44,25 @@ class BlockAndValidity {
}
}
/**
* A test which checks the mempool state (ie defined which transactions should be in memory pool
*/
class MemoryPoolState extends Rule {
Set<InventoryItem> mempool;
public MemoryPoolState(Set<InventoryItem> mempool, String ruleName) {
super(ruleName);
this.mempool = mempool;
}
}
/** An arbitrary rule which the testing client must match */
class Rule {
String ruleName;
Rule(String ruleName) {
this.ruleName = ruleName;
}
}
class TransactionOutPointWithValue {
public TransactionOutPoint outpoint;
public BigInteger value;
@ -52,10 +74,10 @@ class TransactionOutPointWithValue {
}
}
class BlockAndValidityList {
public List<BlockAndValidity> list;
class RuleList {
public List<Rule> list;
public int maximumReorgBlockCount;
public BlockAndValidityList(List<BlockAndValidity> list, int maximumReorgBlockCount) {
public RuleList(List<Rule> list, int maximumReorgBlockCount) {
this.list = list;
this.maximumReorgBlockCount = maximumReorgBlockCount;
}
@ -77,24 +99,24 @@ public class FullBlockTestGenerator {
Utils.rollMockClock(0); // Set a mock clock for timestamp tests
}
public BlockAndValidityList getBlocksToTest(boolean addSigExpensiveBlocks, boolean runLargeReorgs, File blockStorageFile) throws ScriptException, ProtocolException, IOException {
public RuleList getBlocksToTest(boolean addSigExpensiveBlocks, boolean runLargeReorgs, File blockStorageFile) throws ScriptException, ProtocolException, IOException {
final FileOutputStream outStream = blockStorageFile != null ? new FileOutputStream(blockStorageFile) : null;
List<BlockAndValidity> blocks = new LinkedList<BlockAndValidity>() {
List<Rule> blocks = new LinkedList<Rule>() {
@Override
public boolean add(BlockAndValidity element) {
if (outStream != null) {
public boolean add(Rule element) {
if (outStream != null && element instanceof BlockAndValidity) {
try {
outStream.write((int) (params.getPacketMagic() >>> 24));
outStream.write((int) (params.getPacketMagic() >>> 16));
outStream.write((int) (params.getPacketMagic() >>> 8));
outStream.write((int) (params.getPacketMagic() >>> 0));
byte[] block = element.block.bitcoinSerialize();
byte[] block = ((BlockAndValidity)element).block.bitcoinSerialize();
byte[] length = new byte[4];
Utils.uint32ToByteArrayBE(block.length, length, 0);
outStream.write(Utils.reverseBytes(length));
outStream.write(block);
element.block = null;
((BlockAndValidity)element).block = null;
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -102,7 +124,7 @@ public class FullBlockTestGenerator {
return super.add(element);
}
};
BlockAndValidityList ret = new BlockAndValidityList(blocks, 10);
RuleList ret = new RuleList(blocks, 10);
Queue<TransactionOutPointWithValue> spendableOutputs = new LinkedList<TransactionOutPointWithValue>();
@ -1419,29 +1441,93 @@ public class FullBlockTestGenerator {
b76.getTransactions().get(0).getOutputs().get(0).getValue(),
b76.getTransactions().get(0).getOutputs().get(0).getScriptPubKey()));
// Test transaction resurrection
// -> b77 (24) -> b78 (22) -> b79 (23)
// \-> b80 (22) -> b81 (23) -> b82 (24)
// b78 creates a tx, which is spent in b79. after b82, both should be in mempool
//
TransactionOutPointWithValue out24 = spendableOutputs.poll(); Preconditions.checkState(out24 != null);
TransactionOutPointWithValue out25 = spendableOutputs.poll(); Preconditions.checkState(out25 != null);
TransactionOutPointWithValue out26 = spendableOutputs.poll(); Preconditions.checkState(out26 != null);
TransactionOutPointWithValue out27 = spendableOutputs.poll(); Preconditions.checkState(out27 != null);
Block b77 = createNextBlock(b76, chainHeadHeight + 25, out24, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b77, true, false, b77.getHash(), chainHeadHeight + 25, "b77"));
Block b78 = createNextBlock(b77, chainHeadHeight + 26, out25, null);
Transaction b78tx = new Transaction(params);
{
b78tx.addOutput(new TransactionOutput(params, b78tx, BigInteger.ZERO, new byte[]{OP_TRUE}));
addOnlyInputToTransaction(b78tx, new TransactionOutPointWithValue(
new TransactionOutPoint(params, 1, b77.getTransactions().get(1).getHash()),
BigInteger.valueOf(1), b77.getTransactions().get(1).getOutputs().get(1).getScriptPubKey()));
b78.addTransaction(b78tx);
}
b78.solve();
blocks.add(new BlockAndValidity(blockToHeightMap, b78, true, false, b78.getHash(), chainHeadHeight + 26, "b78"));
Block b79 = createNextBlock(b78, chainHeadHeight + 27, out26, null);
Transaction b79tx = new Transaction(params);
{
b79tx.addOutput(new TransactionOutput(params, b79tx, BigInteger.ZERO, new byte[]{OP_TRUE}));
b79tx.addInput(new TransactionInput(params, b79tx, new byte[]{OP_TRUE}, new TransactionOutPoint(params, 0, b78tx.getHash())));
b79.addTransaction(b79tx);
}
b79.solve();
blocks.add(new BlockAndValidity(blockToHeightMap, b79, true, false, b79.getHash(), chainHeadHeight + 27, "b79"));
blocks.add(new MemoryPoolState(new HashSet<InventoryItem>(), "post-b79 empty mempool"));
Block b80 = createNextBlock(b77, chainHeadHeight + 26, out25, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b80, true, false, b79.getHash(), chainHeadHeight + 27, "b80"));
spendableOutputs.offer(new TransactionOutPointWithValue(
new TransactionOutPoint(params, 0, b80.getTransactions().get(0).getHash()),
b80.getTransactions().get(0).getOutputs().get(0).getValue(),
b80.getTransactions().get(0).getOutputs().get(0).getScriptPubKey()));
Block b81 = createNextBlock(b80, chainHeadHeight + 27, out26, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b81, true, false, b79.getHash(), chainHeadHeight + 27, "b81"));
spendableOutputs.offer(new TransactionOutPointWithValue(
new TransactionOutPoint(params, 0, b81.getTransactions().get(0).getHash()),
b81.getTransactions().get(0).getOutputs().get(0).getValue(),
b81.getTransactions().get(0).getOutputs().get(0).getScriptPubKey()));
Block b82 = createNextBlock(b81, chainHeadHeight + 28, out27, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b82, true, false, b82.getHash(), chainHeadHeight + 28, "b82"));
spendableOutputs.offer(new TransactionOutPointWithValue(
new TransactionOutPoint(params, 0, b82.getTransactions().get(0).getHash()),
b82.getTransactions().get(0).getOutputs().get(0).getValue(),
b82.getTransactions().get(0).getOutputs().get(0).getScriptPubKey()));
HashSet<InventoryItem> post82Mempool = new HashSet<InventoryItem>();
post82Mempool.add(new InventoryItem(InventoryItem.Type.Transaction, b78tx.getHash()));
post82Mempool.add(new InventoryItem(InventoryItem.Type.Transaction, b79tx.getHash()));
blocks.add(new MemoryPoolState(post82Mempool, "post-b82 tx resurrection"));
// The remaining tests arent designed to fit in the standard flow, and thus must always come last
// Add new tests here.
//TODO: Explicitly address MoneyRange() checks
// Test massive reorgs (in terms of tx count)
// -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) -> b72 (21) -> b1001 (22) -> lots of outputs -> lots of spends
// Reorg back to:
// -> b60 (17) -> b64 (18) -> b65 (19) -> b69 (20) -> b72 (21) -> b1001 (22) -> empty blocks
//
TransactionOutPointWithValue out24 = spendableOutputs.poll(); Preconditions.checkState(out24 != null);
TransactionOutPointWithValue out28 = spendableOutputs.poll(); Preconditions.checkState(out28 != null);
Block b1001 = createNextBlock(b76, chainHeadHeight + 25, out24, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b1001, true, false, b1001.getHash(), chainHeadHeight + 25, "b1001"));
Block b1001 = createNextBlock(b82, chainHeadHeight + 29, out28, null);
blocks.add(new BlockAndValidity(blockToHeightMap, b1001, true, false, b1001.getHash(), chainHeadHeight + 29, "b1001"));
spendableOutputs.offer(new TransactionOutPointWithValue(
new TransactionOutPoint(params, 0, b1001.getTransactions().get(0).getHash()),
b1001.getTransactions().get(0).getOutputs().get(0).getValue(),
b1001.getTransactions().get(0).getOutputs().get(0).getScriptPubKey()));
int nextHeight = chainHeadHeight + 30;
if (runLargeReorgs) {
// No way you can fit this test in memory
Preconditions.checkArgument(blockStorageFile != null);
Block lastBlock = b1001;
int nextHeight = chainHeadHeight + 26;
TransactionOutPoint lastOutput = new TransactionOutPoint(params, 2, b1001.getTransactions().get(1).getHash());
int blockCountAfter1001;
@ -1522,8 +1608,6 @@ public class FullBlockTestGenerator {
ret.maximumReorgBlockCount = Math.max(ret.maximumReorgBlockCount, blockCountAfter1001);
}
//TODO: Explicitly address MoneyRange() checks
if (outStream != null)
outStream.close();

View File

@ -64,39 +64,42 @@ public class FullPrunedBlockChainTest {
public void testGeneratedChain() throws Exception {
// Tests various test cases from FullBlockTestGenerator
FullBlockTestGenerator generator = new FullBlockTestGenerator(params);
BlockAndValidityList blockList = generator.getBlocksToTest(false, false, null);
RuleList blockList = generator.getBlocksToTest(false, false, null);
store = new MemoryFullPrunedBlockStore(params, blockList.maximumReorgBlockCount);
chain = new FullPrunedBlockChain(params, store);
for (BlockAndValidity block : blockList.list) {
for (Rule rule : blockList.list) {
if (!(rule instanceof BlockAndValidity))
continue;
BlockAndValidity block = (BlockAndValidity) rule;
boolean threw = false;
try {
if (chain.add(block.block) != block.connects) {
log.error("Block didn't match connects flag on block " + block.blockName);
log.error("Block didn't match connects flag on block " + block.ruleName);
fail();
}
} catch (VerificationException e) {
threw = true;
if (!block.throwsException) {
log.error("Block didn't match throws flag on block " + block.blockName);
log.error("Block didn't match throws flag on block " + block.ruleName);
throw e;
}
if (block.connects) {
log.error("Block didn't match connects flag on block " + block.blockName);
log.error("Block didn't match connects flag on block " + block.ruleName);
fail();
}
}
if (!threw && block.throwsException) {
log.error("Block didn't match throws flag on block " + block.blockName);
log.error("Block didn't match throws flag on block " + block.ruleName);
fail();
}
if (!chain.getChainHead().getHeader().getHash().equals(block.hashChainTipAfterBlock)) {
log.error("New block head didn't match the correct value after block " + block.blockName);
log.error("New block head didn't match the correct value after block " + block.ruleName);
fail();
}
if (chain.getChainHead().getHeight() != block.heightAfterBlock) {
log.error("New block head didn't match the correct height after block " + block.blockName);
log.error("New block head didn't match the correct height after block " + block.ruleName);
fail();
}
}