From 3e5e3496ea3d57992597442219fc99232e8d0517 Mon Sep 17 00:00:00 2001 From: Andreas Schildbach Date: Sun, 7 Sep 2014 00:19:08 +0200 Subject: [PATCH] Introduce textual checkpoint format. It's basically one base64-encoded line per checkpoint. The BuildCheckpoints tool now generates and sanity checks both formats. --- .../bitcoin/core/CheckpointManager.java | 68 +++++++++++++++++-- .../bitcoin/tools/BuildCheckpoints.java | 51 +++++++++++--- 2 files changed, 104 insertions(+), 15 deletions(-) diff --git a/core/src/main/java/com/google/bitcoin/core/CheckpointManager.java b/core/src/main/java/com/google/bitcoin/core/CheckpointManager.java index c5369332..06459e4e 100644 --- a/core/src/main/java/com/google/bitcoin/core/CheckpointManager.java +++ b/core/src/main/java/com/google/bitcoin/core/CheckpointManager.java @@ -19,14 +19,23 @@ package com.google.bitcoin.core; import com.google.bitcoin.store.BlockStore; import com.google.bitcoin.store.BlockStoreException; import com.google.bitcoin.store.FullPrunedBlockStore; +import com.google.common.base.Charsets; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.common.hash.Hashing; +import com.google.common.io.BaseEncoding; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.security.DigestInputStream; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; @@ -62,6 +71,8 @@ import static com.google.common.base.Preconditions.*; public class CheckpointManager { private static final Logger log = LoggerFactory.getLogger(CheckpointManager.class); + private static final String BINARY_MAGIC = "CHECKPOINTS 1"; + private static final String TEXTUAL_MAGIC = "TXT CHECKPOINTS 1"; private static final int MAX_SIGNATURES = 256; // Map of block header time to data. @@ -70,19 +81,33 @@ public class CheckpointManager { protected final NetworkParameters params; protected final Sha256Hash dataHash; + public static final BaseEncoding BASE64 = BaseEncoding.base64().omitPadding(); + public CheckpointManager(NetworkParameters params, InputStream inputStream) throws IOException { this.params = checkNotNull(params); checkNotNull(inputStream); + inputStream = new BufferedInputStream(inputStream); + inputStream.mark(1); + int first = inputStream.read(); + inputStream.reset(); + if (first == BINARY_MAGIC.charAt(0)) + dataHash = readBinary(inputStream); + else if (first == TEXTUAL_MAGIC.charAt(0)) + dataHash = readTextual(inputStream); + else + throw new IOException("Unsupported format."); + } + + private Sha256Hash readBinary(InputStream inputStream) throws IOException { DataInputStream dis = null; try { MessageDigest digest = MessageDigest.getInstance("SHA-256"); DigestInputStream digestInputStream = new DigestInputStream(inputStream, digest); dis = new DataInputStream(digestInputStream); digestInputStream.on(false); - String magic = "CHECKPOINTS 1"; - byte[] header = new byte[magic.length()]; + byte[] header = new byte[BINARY_MAGIC.length()]; dis.readFully(header); - if (!Arrays.equals(header, magic.getBytes("US-ASCII"))) + if (!Arrays.equals(header, BINARY_MAGIC.getBytes("US-ASCII"))) throw new IOException("Header bytes did not match expected version"); int numSignatures = checkPositionIndex(dis.readInt(), MAX_SIGNATURES, "Num signatures out of range"); for (int i = 0; i < numSignatures; i++) { @@ -102,8 +127,9 @@ public class CheckpointManager { buffer.position(0); checkpoints.put(block.getHeader().getTimeSeconds(), block); } - dataHash = new Sha256Hash(digest.digest()); + Sha256Hash dataHash = new Sha256Hash(digest.digest()); log.info("Read {} checkpoints, hash is {}", checkpoints.size(), dataHash); + return dataHash; } catch (NoSuchAlgorithmException e) { throw new RuntimeException(e); // Cannot happen. } catch (ProtocolException e) { @@ -114,6 +140,40 @@ public class CheckpointManager { } } + private Sha256Hash readTextual(InputStream inputStream) throws IOException { + Hasher hasher = Hashing.sha256().newHasher(); + BufferedReader reader = null; + try { + reader = new BufferedReader(new InputStreamReader(inputStream, Charsets.US_ASCII)); + String magic = reader.readLine(); + if (!TEXTUAL_MAGIC.equals(magic)) + throw new IOException("unexpected magic: " + magic); + int numSigs = Integer.parseInt(reader.readLine()); + for (int i = 0; i < numSigs; i++) + reader.readLine(); // Skip sigs for now. + int numCheckpoints = Integer.parseInt(reader.readLine()); + checkState(numCheckpoints > 0); + // Hash numCheckpoints in a way compatible to the binary format. + hasher.putBytes(ByteBuffer.allocate(4).order(ByteOrder.BIG_ENDIAN).putInt(numCheckpoints).array()); + final int size = StoredBlock.COMPACT_SERIALIZED_SIZE; + ByteBuffer buffer = ByteBuffer.allocate(size); + for (int i = 0; i < numCheckpoints; i++) { + byte[] bytes = BASE64.decode(reader.readLine()); + hasher.putBytes(bytes); + buffer.position(0); + buffer.put(bytes); + buffer.position(0); + StoredBlock block = StoredBlock.deserializeCompact(params, buffer); + checkpoints.put(block.getHeader().getTimeSeconds(), block); + } + HashCode hash = hasher.hash(); + log.info("Read {} checkpoints, hash is {}", checkpoints.size(), hash); + return new Sha256Hash(hash.asBytes()); + } finally { + if (reader != null) reader.close(); + } + } + /** * Returns a {@link StoredBlock} representing the last checkpoint before the given time, for example, normally * you would want to know the checkpoint before the earliest wallet birthday. diff --git a/tools/src/main/java/com/google/bitcoin/tools/BuildCheckpoints.java b/tools/src/main/java/com/google/bitcoin/tools/BuildCheckpoints.java index 489fc78a..90924222 100644 --- a/tools/src/main/java/com/google/bitcoin/tools/BuildCheckpoints.java +++ b/tools/src/main/java/com/google/bitcoin/tools/BuildCheckpoints.java @@ -23,11 +23,15 @@ import com.google.bitcoin.store.BlockStore; import com.google.bitcoin.store.MemoryBlockStore; import com.google.bitcoin.utils.BriefLogFormatter; import com.google.bitcoin.utils.Threading; +import com.google.common.base.Charsets; import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; import java.net.InetAddress; import java.nio.ByteBuffer; import java.security.DigestOutputStream; @@ -44,12 +48,13 @@ import static com.google.common.base.Preconditions.checkState; public class BuildCheckpoints { private static final NetworkParameters PARAMS = MainNetParams.get(); - private static final File CHECKPOINTS_FILE = new File("checkpoints"); + private static final File PLAIN_CHECKPOINTS_FILE = new File("checkpoints"); + private static final File TEXTUAL_CHECKPOINTS_FILE = new File("checkpoints.txt"); public static void main(String[] args) throws Exception { BriefLogFormatter.init(); - // Sorted map of UNIX time of block to StoredBlock object. + // Sorted map of block height to StoredBlock object. final TreeMap checkpoints = new TreeMap(); // Configure bitcoinj to fetch only headers, not save them to disk, connect to a local fully synced/validated @@ -82,7 +87,20 @@ public class BuildCheckpoints { checkState(checkpoints.size() > 0); // Write checkpoint data out. - final FileOutputStream fileOutputStream = new FileOutputStream(CHECKPOINTS_FILE, false); + writeBinaryCheckpoints(checkpoints, PLAIN_CHECKPOINTS_FILE); + writeTextualCheckpoints(checkpoints, TEXTUAL_CHECKPOINTS_FILE); + + peerGroup.stopAsync(); + peerGroup.awaitTerminated(); + store.close(); + + // Sanity check the created files. + sanityCheck(PLAIN_CHECKPOINTS_FILE, checkpoints.size()); + sanityCheck(TEXTUAL_CHECKPOINTS_FILE, checkpoints.size()); + } + + private static void writeBinaryCheckpoints(TreeMap checkpoints, File file) throws Exception { + final FileOutputStream fileOutputStream = new FileOutputStream(file, false); MessageDigest digest = MessageDigest.getInstance("SHA-256"); final DigestOutputStream digestOutputStream = new DigestOutputStream(fileOutputStream, digest); digestOutputStream.on(false); @@ -102,14 +120,27 @@ public class BuildCheckpoints { System.out.println("Hash of checkpoints data is " + checkpointsHash); digestOutputStream.close(); fileOutputStream.close(); + System.out.println("Checkpoints written to '" + file.getCanonicalPath() + "'."); + } - peerGroup.stopAsync(); - peerGroup.awaitTerminated(); - store.close(); + private static void writeTextualCheckpoints(TreeMap checkpoints, File file) throws IOException { + PrintWriter writer = new PrintWriter(new OutputStreamWriter(new FileOutputStream(file), Charsets.US_ASCII)); + writer.println("TXT CHECKPOINTS 1"); + writer.println("0"); // Number of signatures to read. Do this later. + writer.println(checkpoints.size()); + ByteBuffer buffer = ByteBuffer.allocate(StoredBlock.COMPACT_SERIALIZED_SIZE); + for (StoredBlock block : checkpoints.values()) { + block.serializeCompact(buffer); + writer.println(CheckpointManager.BASE64.encode(buffer.array())); + buffer.position(0); + } + writer.close(); + System.out.println("Checkpoints written to '" + file.getCanonicalPath() + "'."); + } - // Sanity check the created file. - CheckpointManager manager = new CheckpointManager(PARAMS, new FileInputStream(CHECKPOINTS_FILE)); - checkState(manager.numCheckpoints() == checkpoints.size()); + private static void sanityCheck(File file, int expectedSize) throws IOException { + CheckpointManager manager = new CheckpointManager(PARAMS, new FileInputStream(file)); + checkState(manager.numCheckpoints() == expectedSize); if (PARAMS.getId().equals(NetworkParameters.ID_MAINNET)) { StoredBlock test = manager.getCheckpointBefore(1390500000); // Thu Jan 23 19:00:00 CET 2014 @@ -122,7 +153,5 @@ public class BuildCheckpoints { checkState(test.getHeader().getHashAsString() .equals("0000000000035ae7d5025c2538067fe7adb1cf5d5d9c31b024137d9090ed13a9")); } - - System.out.println("Checkpoints written to '" + CHECKPOINTS_FILE.getCanonicalPath() + "'."); } }