From 729f2f53120b8083507a56934ef356425d8d12a1 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 25 Apr 2013 13:37:54 -0400 Subject: [PATCH] Store blocks on disk during BitcoindComparisonTool to save memory. --- .../bitcoin/core/BitcoindComparisonTool.java | 28 +++++++++++--- .../bitcoin/core/FullBlockTestGenerator.java | 37 +++++++++++++++++-- .../core/FullPrunedBlockChainTest.java | 2 +- 3 files changed, 57 insertions(+), 10 deletions(-) diff --git a/core/src/test/java/com/google/bitcoin/core/BitcoindComparisonTool.java b/core/src/test/java/com/google/bitcoin/core/BitcoindComparisonTool.java index c2e32529..94a19660 100644 --- a/core/src/test/java/com/google/bitcoin/core/BitcoindComparisonTool.java +++ b/core/src/test/java/com/google/bitcoin/core/BitcoindComparisonTool.java @@ -18,14 +18,21 @@ package com.google.bitcoin.core; import com.google.bitcoin.store.BlockStoreException; import com.google.bitcoin.store.FullPrunedBlockStore; +import com.google.bitcoin.store.H2FullPrunedBlockStore; import com.google.bitcoin.store.MemoryFullPrunedBlockStore; +import com.google.bitcoin.utils.BlockFileLoader; import com.google.bitcoin.utils.BriefLogFormatter; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.math.BigInteger; import java.net.InetAddress; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.LinkedList; import java.util.List; /** @@ -45,8 +52,8 @@ public class BitcoindComparisonTool { public static void main(String[] args) throws Exception { BriefLogFormatter.init(); - System.out.println("USAGE: runLargeReorgs(1/0)"); - boolean runLargeReorgs = Integer.parseInt(args[0]) == 1; + System.out.println("USAGE: bitcoinjBlockStoreLocation runLargeReorgs(1/0)"); + boolean runLargeReorgs = Integer.parseInt(args[1]) == 1; params = NetworkParameters.testNet2(); /** @@ -70,14 +77,20 @@ public class BitcoindComparisonTool { params.genesisBlock.setDifficultyTarget(0x207fFFFFL); // Also set block.nTime = 1296688602; in the same block + File blockFile = File.createTempFile("testBlocks", ".dat"); + blockFile.deleteOnExit(); + FullBlockTestGenerator generator = new FullBlockTestGenerator(params); - BlockAndValidityList blockList = generator.getBlocksToTest(true, runLargeReorgs); + BlockAndValidityList blockList = generator.getBlocksToTest(true, runLargeReorgs, blockFile); + Iterator blocks = new BlockFileLoader(params, Arrays.asList(blockFile)); // Only needs to be set in bitcoinj params.allowEmptyPeerChains = true; try { - store = new MemoryFullPrunedBlockStore(params, blockList.maximumReorgBlockCount); + store = new H2FullPrunedBlockStore(params, args[0], blockList.maximumReorgBlockCount); + ((H2FullPrunedBlockStore)store).resetStore(); + //store = new MemoryFullPrunedBlockStore(params, blockList.maximumReorgBlockCount); chain = new FullPrunedBlockChain(params, store); } catch (BlockStoreException e) { e.printStackTrace(); @@ -138,8 +151,9 @@ public class BitcoindComparisonTool { int invalidBlocks = 0; for (BlockAndValidity block : blockList.list) { boolean threw = false; + Block nextBlock = blocks.next(); try { - if (chain.add(block.block) != block.connects) { + if (chain.add(nextBlock) != block.connects) { log.error("Block didn't match connects flag on block \"" + block.blockName + "\""); invalidBlocks++; } @@ -147,9 +161,11 @@ public class BitcoindComparisonTool { threw = true; if (!block.throwsException) { log.error("Block didn't match throws flag on block \"" + block.blockName + "\""); + e.printStackTrace(); invalidBlocks++; } else if (block.connects) { log.error("Block didn't match connects flag on block \"" + block.blockName + "\""); + e.printStackTrace(); invalidBlocks++; } } @@ -164,7 +180,7 @@ public class BitcoindComparisonTool { invalidBlocks++; } - bitcoind.sendMessage(block.block); + bitcoind.sendMessage(nextBlock); locator.clear(); locator.add(bitcoindChainHead); bitcoind.sendMessage(new GetHeadersMessage(params, locator, hashTo)); diff --git a/core/src/test/java/com/google/bitcoin/core/FullBlockTestGenerator.java b/core/src/test/java/com/google/bitcoin/core/FullBlockTestGenerator.java index bf245d60..70da5a0a 100644 --- a/core/src/test/java/com/google/bitcoin/core/FullBlockTestGenerator.java +++ b/core/src/test/java/com/google/bitcoin/core/FullBlockTestGenerator.java @@ -6,6 +6,8 @@ import com.google.bitcoin.script.ScriptBuilder; import com.google.common.base.Preconditions; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; import java.math.BigInteger; import java.util.*; @@ -75,8 +77,31 @@ public class FullBlockTestGenerator { Utils.rollMockClock(0); // Set a mock clock for timestamp tests } - public BlockAndValidityList getBlocksToTest(boolean addSigExpensiveBlocks, boolean runLargeReorgs) throws ScriptException, ProtocolException, IOException { - List blocks = new LinkedList(); + public BlockAndValidityList getBlocksToTest(boolean addSigExpensiveBlocks, boolean runLargeReorgs, File blockStorageFile) throws ScriptException, ProtocolException, IOException { + final FileOutputStream outStream = blockStorageFile != null ? new FileOutputStream(blockStorageFile) : null; + + List blocks = new LinkedList() { + @Override + public boolean add(BlockAndValidity element) { + if (outStream != null) { + try { + outStream.write((int) (params.packetMagic >>> 24)); + outStream.write((int) (params.packetMagic >>> 16)); + outStream.write((int) (params.packetMagic >>> 8)); + outStream.write((int) (params.packetMagic >>> 0)); + byte[] block = element.block.bitcoinSerialize(); + byte[] length = new byte[4]; + Utils.uint32ToByteArrayBE(block.length, length, 0); + outStream.write(Utils.reverseBytes(length)); + outStream.write(block); + element.block = null; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + return super.add(element); + } + }; BlockAndValidityList ret = new BlockAndValidityList(blocks, 10); Queue spendableOutputs = new LinkedList(); @@ -1305,13 +1330,16 @@ public class FullBlockTestGenerator { b73.getTransactions().get(0).getOutputs().get(0).getScriptPubKey())); if (runLargeReorgs) { + // No way you can fit this test in memory + Preconditions.checkArgument(blockStorageFile != null); + Block lastBlock = b73; int nextHeight = chainHeadHeight + 24; TransactionOutPoint lastOutput = new TransactionOutPoint(params, 2, b73.getTransactions().get(1).getHash()); int blockCountAfter73; List hashesToSpend = new LinkedList(); // all index 0 - final int TRANSACTION_CREATION_BLOCKS = 50; + final int TRANSACTION_CREATION_BLOCKS = 100; for (blockCountAfter73 = 0; blockCountAfter73 < TRANSACTION_CREATION_BLOCKS; blockCountAfter73++) { Block block = createNextBlock(lastBlock, nextHeight++, null, null); while (block.getMessageSize() < Block.MAX_BLOCK_SIZE - 500) { @@ -1389,6 +1417,9 @@ public class FullBlockTestGenerator { //TODO: Explicitly address MoneyRange() checks + if (outStream != null) + outStream.close(); + // (finally) return the created chain return ret; } diff --git a/core/src/test/java/com/google/bitcoin/core/FullPrunedBlockChainTest.java b/core/src/test/java/com/google/bitcoin/core/FullPrunedBlockChainTest.java index 2c319400..bd7b6aa8 100644 --- a/core/src/test/java/com/google/bitcoin/core/FullPrunedBlockChainTest.java +++ b/core/src/test/java/com/google/bitcoin/core/FullPrunedBlockChainTest.java @@ -65,7 +65,7 @@ public class FullPrunedBlockChainTest { public void testGeneratedChain() throws Exception { // Tests various test cases from FullBlockTestGenerator FullBlockTestGenerator generator = new FullBlockTestGenerator(unitTestParams); - BlockAndValidityList blockList = generator.getBlocksToTest(false, false); + BlockAndValidityList blockList = generator.getBlocksToTest(false, false, null); store = new MemoryFullPrunedBlockStore(unitTestParams, blockList.maximumReorgBlockCount); chain = new FullPrunedBlockChain(unitTestParams, store);