diff --git a/core/src/main/java/com/google/bitcoin/store/FullPrunedBlockStore.java b/core/src/main/java/com/google/bitcoin/store/FullPrunedBlockStore.java new file mode 100644 index 00000000..a60a2e34 --- /dev/null +++ b/core/src/main/java/com/google/bitcoin/store/FullPrunedBlockStore.java @@ -0,0 +1,94 @@ +/* + * Copyright 2012 Matt Corallo. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.bitcoin.store; + +import com.google.bitcoin.core.Sha256Hash; +import com.google.bitcoin.core.StoredBlock; +import com.google.bitcoin.core.StoredTransactionOutput; +import com.google.bitcoin.core.StoredUndoableBlock; + +/** + *

An implementor of FullPrunedBlockStore saves StoredBlock objects to some storage mechanism.

+ * + *

It should store the {@link StoredUndoableBlock}s of a number of recent blocks. + * It is advisable to store any {@link StoredUndoableBlock} which has a height > head.height - N. + * Because N determines the memory usage, it is recommended that N be customizable. N should be chosen such that + * re-orgs beyond that point are vanishingly unlikely, for example, a few thousand blocks is a reasonable choice.

+ * + *

It must store the {@link StoredBlock} of all blocks.

+ * + *

A FullPrunedBlockStore contains a map of hashes to [Full]StoredBlock. The hash is the double digest of the + * Bitcoin serialization of the block header, not the header with the extra data as well.

+ * + *

A FullPrunedBlockStore also contains a map of hash+index to StoredTransactionOutput. Again, the hash is + * a standard Bitcoin double-SHA256 hash of the transaction.

+ * + *

FullPrunedBlockStores are thread safe.

+ */ +public interface FullPrunedBlockStore extends BlockStore { + /** + * Saves the given {@link StoredUndoableBlock} and {@link StoredBlock}. Calculates keys from the {@link StoredBlock} + * Note that a call to put(StoredBlock) will throw a BlockStoreException if its height is > head.height - N + * @throws BlockStoreException if there is a problem with the underlying storage layer, such as running out of disk space. + */ + void put(StoredBlock storedBlock, StoredUndoableBlock undoableBlock) throws BlockStoreException; + + /** + * Returns a {@link StoredUndoableBlock} who's block.getHash() method will be equal to the + * parameter. If no such block is found, returns null. + * Note that this may return null more often than get(Sha256Hash hash) as not all {@link StoredBlock}s have a + * {@link StoredUndoableBlock} copy stored as well. + */ + StoredUndoableBlock getUndoBlock(Sha256Hash hash) throws BlockStoreException; + + /** + * Gets a {@link StoredTransactionOutput} with the given hash and index, or null if none is found + */ + StoredTransactionOutput getTransactionOutput(Sha256Hash hash, long index) throws BlockStoreException; + + /** + * Adds a {@link StoredTransactionOutput} to the list of unspent TransactionOutputs + */ + void addUnspentTransactionOutput(StoredTransactionOutput out) throws BlockStoreException; + + /** + * Removes a {@link StoredTransactionOutput} from the list of unspent TransactionOutputs + * @throws BlockStoreException if there is an underlying storage issue, or out was not in the list. + */ + void removeUnspentTransactionOutput(StoredTransactionOutput out) throws BlockStoreException; + + /** + * True if this store has any unspent outputs from a transaction with a hash equal to the first parameter + * @param numOutputs the number of outputs the given transaction has + */ + boolean hasUnspentOutputs(Sha256Hash hash, int numOutputs) throws BlockStoreException; + + /** + *

Begins/Commits/Aborts a database transaction.

+ * + *

If abortDatabaseBatchWrite() is called by the same thread that called beginDatabaseBatchWrite(), + * any data writes between this call and abortDatabaseBatchWrite() made by the same thread + * should be discarded.

+ * + *

Furthermore, any data written after a call to beginDatabaseBatchWrite() should not be readable + * by any other threads until commitDatabaseBatchWrite() has been called by this thread. + * Multiple calls to beginDatabaseBatchWrite() in any given thread should be ignored and treated as one call.

+ */ + void beginDatabaseBatchWrite() throws BlockStoreException; + void commitDatabaseBatchWrite() throws BlockStoreException; + void abortDatabaseBatchWrite() throws BlockStoreException; +} diff --git a/core/src/main/java/com/google/bitcoin/store/MemoryFullPrunedBlockStore.java b/core/src/main/java/com/google/bitcoin/store/MemoryFullPrunedBlockStore.java new file mode 100644 index 00000000..e3a3165c --- /dev/null +++ b/core/src/main/java/com/google/bitcoin/store/MemoryFullPrunedBlockStore.java @@ -0,0 +1,339 @@ +/* + * Copyright 2012 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.bitcoin.store; + +import com.google.bitcoin.core.*; +import com.google.common.base.Objects; +import com.google.common.base.Preconditions; + +import java.io.Serializable; +import java.util.*; + + +/** + * Used as a key for memory map (to avoid having to think about NetworkParameters, + * which is required for {@link TransactionOutPoint} + */ +class StoredTransactionOutPoint implements Serializable { + private static final long serialVersionUID = -4064230006297064377L; + + /** Hash of the transaction to which we refer. */ + Sha256Hash hash; + /** Which output of that transaction we are talking about. */ + long index; + + StoredTransactionOutPoint(Sha256Hash hash, long index) { + this.hash = hash; + this.index = index; + } + + StoredTransactionOutPoint(StoredTransactionOutput out) { + this.hash = out.getHash(); + this.index = out.getIndex(); + } + + /** + * The hash of the transaction to which we refer + */ + Sha256Hash getHash() { + return hash; + } + + /** + * The index of the output in transaction to which we refer + */ + long getIndex() { + return index; + } + + public int hashCode() { + return this.hash.hashCode() + (int)index; + } + + public String toString() { + return "Stored transaction out point: " + hash.toString() + ":" + index; + } + + public boolean equals(Object o) { + if (!(o instanceof StoredTransactionOutPoint)) return false; + return ((StoredTransactionOutPoint)o).getIndex() == this.index && + Objects.equal(this.getHash(), ((StoredTransactionOutPoint)o).getHash()); + } +} + +/** + * A HashMap that is DB transaction-aware + * This class is not thread-safe. + */ +class TransactionalHashMap { + ThreadLocal> tempMap; + ThreadLocal> tempSetRemoved; + private ThreadLocal inTransaction; + + HashMap map; + + public TransactionalHashMap() { + tempMap = new ThreadLocal>(); + tempSetRemoved = new ThreadLocal>(); + inTransaction = new ThreadLocal(); + map = new HashMap(); + } + + public void beginDatabaseBatchWrite() { + inTransaction.set(true); + } + + public void commitDatabaseBatchWrite() { + if (tempSetRemoved.get() != null) + for(KeyType key : tempSetRemoved.get()) + map.remove(key); + if (tempMap.get() != null) + for (Map.Entry entry : tempMap.get().entrySet()) + map.put(entry.getKey(), entry.getValue()); + abortDatabaseBatchWrite(); + } + + public void abortDatabaseBatchWrite() { + inTransaction.set(false); + tempSetRemoved.remove(); + tempMap.remove(); + } + + public ValueType get(KeyType key) { + if (Boolean.TRUE.equals(inTransaction.get())) { + if (tempMap.get() != null) { + ValueType value = tempMap.get().get(key); + if (value != null) + return value; + } + if (tempSetRemoved.get() != null && tempSetRemoved.get().contains(key)) + return null; + } + return map.get(key); + } + + public void put(KeyType key, ValueType value) { + if (Boolean.TRUE.equals(inTransaction.get())) { + if (tempSetRemoved.get() != null) + tempSetRemoved.get().remove(key); + if (tempMap.get() == null) + tempMap.set(new HashMap()); + tempMap.get().put(key, value); + }else{ + map.put(key, value); + } + } + + public ValueType remove(KeyType key) { + if (Boolean.TRUE.equals(inTransaction.get())) { + ValueType retVal = map.get(key); + if (retVal != null) { + if (tempSetRemoved.get() == null) + tempSetRemoved.set(new HashSet()); + tempSetRemoved.get().add(key); + } + if (tempMap.get() != null) { + ValueType tempVal = tempMap.get().remove(key); + if (tempVal != null) + return tempVal; + } + return retVal; + }else{ + return map.remove(key); + } + } +} + +/** + * A Map with multiple key types that is DB per-thread-transaction-aware. + * However, this class is not thread-safe. + * @param UniqueKeyType is a key that must be unique per object + * @param MultiKeyType is a key that can have multiple values + */ +class TransactionalMultiKeyHashMap { + TransactionalHashMap mapValues; + HashMap> mapKeys; + + public TransactionalMultiKeyHashMap() { + mapValues = new TransactionalHashMap(); + mapKeys = new HashMap>(); + } + + public void BeginTransaction() { + mapValues.beginDatabaseBatchWrite(); + } + + public void CommitTransaction() { + mapValues.commitDatabaseBatchWrite(); + } + + public void AbortTransaction() { + mapValues.abortDatabaseBatchWrite(); + } + + public ValueType get(UniqueKeyType key) { + return mapValues.get(key); + } + + public void put(UniqueKeyType uniqueKey, MultiKeyType multiKey, ValueType value) { + mapValues.put(uniqueKey, value); + Set set = mapKeys.get(multiKey); + if (set == null) { + set = new HashSet(); + set.add(uniqueKey); + mapKeys.put(multiKey, set); + }else{ + set.add(uniqueKey); + } + } + + public ValueType removeByUniqueKey(UniqueKeyType key) { + return mapValues.remove(key); + } + + public void removeByMultiKey(MultiKeyType key) { + Set set = mapKeys.remove(key); + if (set != null) + for (UniqueKeyType uniqueKey : set) + removeByUniqueKey(uniqueKey); + } +} + +/** + * Keeps {@link StoredBlock}s, {@link StoredUndoableBlock}s and {@link StoredTransactionOutput}s in memory. + * Used primarily for unit testing. + */ +public class MemoryFullPrunedBlockStore implements FullPrunedBlockStore { + private TransactionalHashMap blockMap; + private TransactionalMultiKeyHashMap fullBlockMap; + //TODO: Use something more suited to remove-heavy use? + private TransactionalHashMap transactionOutputMap; + private StoredBlock chainHead; + private int fullStoreDepth; + + /** + * Set up the MemoryFullPrunedBlockStore + * @param params The network parameters of this block store - used to get genesis block + * @param fullStoreDepth The depth of blocks to keep FullStoredBlocks instead of StoredBlocks + */ + public MemoryFullPrunedBlockStore(NetworkParameters params, int fullStoreDepth) { + blockMap = new TransactionalHashMap(); + fullBlockMap = new TransactionalMultiKeyHashMap(); + transactionOutputMap = new TransactionalHashMap(); + this.fullStoreDepth = fullStoreDepth > 0 ? fullStoreDepth : 1; + // Insert the genesis block. + try { + StoredBlock storedGenesisHeader = new StoredBlock(params.genesisBlock.cloneAsHeader(), params.genesisBlock.getWork(), 0); + + LinkedList genesisTransactions = new LinkedList(); + for (Transaction tx : params.genesisBlock.getTransactions()) + genesisTransactions.add(new StoredTransaction(tx, 0)); + StoredUndoableBlock storedGenesis = new StoredUndoableBlock(params.genesisBlock.getHash(), genesisTransactions); + + put(storedGenesisHeader, storedGenesis); + setChainHead(storedGenesisHeader); + } catch (BlockStoreException e) { + throw new RuntimeException(e); // Cannot happen. + } catch (VerificationException e) { + throw new RuntimeException(e); // Cannot happen. + } + } + + public synchronized void put(StoredBlock block) throws BlockStoreException { + Preconditions.checkNotNull(blockMap, "MemoryFullPrunedBlockStore is closed"); + if (block.getHeight() > chainHead.getHeight() - fullStoreDepth) + throw new BlockStoreException("Putting a StoredBlock in MemoryFullPrunedBlockStore at height higher than head - fullStoredDepth"); + Sha256Hash hash = block.getHeader().getHash(); + blockMap.put(hash, block); + } + + public synchronized void put(StoredBlock storedBlock, StoredUndoableBlock undoableBlock) throws BlockStoreException { + Preconditions.checkNotNull(blockMap, "MemoryFullPrunedBlockStore is closed"); + fullBlockMap.put(storedBlock.getHeader().getHash(), storedBlock.getHeight(), undoableBlock); + blockMap.put(storedBlock.getHeader().getHash(), storedBlock); + } + + public synchronized StoredBlock get(Sha256Hash hash) throws BlockStoreException { + Preconditions.checkNotNull(blockMap, "MemoryFullPrunedBlockStore is closed"); + return blockMap.get(hash); + } + + public synchronized StoredUndoableBlock getUndoBlock(Sha256Hash hash) throws BlockStoreException { + Preconditions.checkNotNull(fullBlockMap, "MemoryFullPrunedBlockStore is closed"); + return fullBlockMap.get(hash); + } + + public StoredBlock getChainHead() throws BlockStoreException { + Preconditions.checkNotNull(blockMap, "MemoryFullPrunedBlockStore is closed"); + return chainHead; + } + + public synchronized void setChainHead(StoredBlock chainHead) throws BlockStoreException { + Preconditions.checkNotNull(blockMap, "MemoryFullPrunedBlockStore is closed"); + this.chainHead = chainHead; + // Potential leak here if not all blocks get setChainHead'd + // Though the FullPrunedBlockStore allows for this, the current AbstractBlockChain will not do it. + fullBlockMap.removeByMultiKey(chainHead.getHeight() - fullStoreDepth); + } + + public void close() { + blockMap = null; + fullBlockMap = null; + transactionOutputMap = null; + } + + public synchronized StoredTransactionOutput getTransactionOutput(Sha256Hash hash, long index) throws BlockStoreException { + Preconditions.checkNotNull(transactionOutputMap, "MemoryFullPrunedBlockStore is closed"); + return transactionOutputMap.get(new StoredTransactionOutPoint(hash, index)); + } + + public synchronized void addUnspentTransactionOutput(StoredTransactionOutput out) throws BlockStoreException { + Preconditions.checkNotNull(transactionOutputMap, "MemoryFullPrunedBlockStore is closed"); + transactionOutputMap.put(new StoredTransactionOutPoint(out), out); + } + + public synchronized void removeUnspentTransactionOutput(StoredTransactionOutput out) throws BlockStoreException { + Preconditions.checkNotNull(transactionOutputMap, "MemoryFullPrunedBlockStore is closed"); + if (transactionOutputMap.remove(new StoredTransactionOutPoint(out)) == null) + throw new BlockStoreException("Tried to remove a StoredTransactionOutput from MemoryFullPrunedBlockStore that it didn't have!"); + } + + public synchronized void beginDatabaseBatchWrite() throws BlockStoreException { + blockMap.beginDatabaseBatchWrite(); + fullBlockMap.BeginTransaction(); + transactionOutputMap.beginDatabaseBatchWrite(); + } + + public synchronized void commitDatabaseBatchWrite() throws BlockStoreException { + blockMap.commitDatabaseBatchWrite(); + fullBlockMap.CommitTransaction(); + transactionOutputMap.commitDatabaseBatchWrite(); + } + + public synchronized void abortDatabaseBatchWrite() throws BlockStoreException { + blockMap.abortDatabaseBatchWrite(); + fullBlockMap.AbortTransaction(); + transactionOutputMap.abortDatabaseBatchWrite(); + } + + public boolean hasUnspentOutputs(Sha256Hash hash, int numOutputs) throws BlockStoreException { + for (int i = 0; i < numOutputs; i++) + if (getTransactionOutput(hash, i) != null) + return true; + return false; + } +}