Bitcoiny improvements

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -1,6 +1,7 @@
 package org.qortal.crosschain;
 
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -49,8 +50,12 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 
 	protected final NetworkParameters params;
 
-	/** Keys that have been previously partially, or fully, spent */
+	/** Keys that have been previously marked as fully spent,<br>
-	protected final Set<ECKey> spentKeys = new HashSet<>();
+	 * i.e. keys with transactions but with no unspent outputs. */
+	protected final Set<ECKey> spentKeys = Collections.synchronizedSet(new HashSet<>());
+
+	/** How many bitcoinj wallet keys to generate in each batch. */
+	private static final int WALLET_KEY_LOOKAHEAD_INCREMENT = 3;
 
 	/** Byte offset into raw block headers to block timestamp. */
 	private static final int TIMESTAMP_OFFSET = 4 + 32 + 32;
@@ -186,6 +191,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	 * @return list of unspent outputs, or empty list if address unknown
 	 * @throws ForeignBlockchainException if there was an error.
 	 */
+	// TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead
 	public List<TransactionOutput> getUnspentOutputs(String base58Address) throws ForeignBlockchainException {
 		List<UnspentOutput> unspentOutputs = this.blockchain.getUnspentOutputs(addressToScriptPubKey(base58Address), false);
 
@@ -205,10 +211,10 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	 * @return list of outputs, or empty list if transaction unknown
 	 * @throws ForeignBlockchainException if there was an error.
 	 */
+	// TODO: don't return bitcoinj-based objects like TransactionOutput, use BitcoinyTransaction.Output instead
 	public List<TransactionOutput> getOutputs(byte[] txHash) throws ForeignBlockchainException {
 		byte[] rawTransactionBytes = this.blockchain.getRawTransaction(txHash);
 
-		// XXX bitcoinj: replace with getTransaction() below
 		Context.propagate(bitcoinjContext);
 		Transaction transaction = new Transaction(this.params, rawTransactionBytes);
 		return transaction.getOutputs();
@@ -313,14 +319,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	public Long getWalletBalance(String key58) {
 		Context.propagate(bitcoinjContext);
 
-		final DeterministicKey watchKey = DeterministicKey.deserializeB58(null, key58, this.params);
+		Wallet wallet = walletFromDeterministicKey58(key58);
-
-		Wallet wallet;
-		if (watchKey.hasPrivKey())
-			wallet = Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
-		else
-			wallet = Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
-
 		wallet.setUTXOProvider(new WalletAwareUTXOProvider(this, wallet));
 
 		Coin balance = wallet.getBalance();
@@ -333,17 +332,10 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	public Set<BitcoinyTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
 		Context.propagate(bitcoinjContext);
 
-		final DeterministicKey watchKey = DeterministicKey.deserializeB58(null, key58, this.params);
+		Wallet wallet = walletFromDeterministicKey58(key58);
-
-		Wallet wallet;
-		if (watchKey.hasPrivKey())
-			wallet = Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
-		else
-			wallet = Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
-
 		DeterministicKeyChain keyChain = wallet.getActiveKeyChain();
 
-		keyChain.setLookaheadSize(WalletAwareUTXOProvider.LOOKAHEAD_INCREMENT);
+		keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
 		keyChain.maybeLookAhead();
 
 		List<DeterministicKey> keys = new ArrayList<>(keyChain.getLeafKeys());
@@ -372,41 +364,31 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 				}
 			}
 
-			if (!areAllKeysUnused) {
+			if (areAllKeysUnused)
-				// Generate some more keys
+				// No transactions for this batch of keys so assume we're done searching.
-				keyChain.setLookaheadSize(keyChain.getLookaheadSize() + WalletAwareUTXOProvider.LOOKAHEAD_INCREMENT);
+				return walletTransactions;
-				keyChain.maybeLookAhead();
 
-				// This returns all keys, including those already in 'keys'
+			// Generate some more keys
-				List<DeterministicKey> allLeafKeys = keyChain.getLeafKeys();
+			keys.addAll(generateMoreKeys(keyChain));
-				// Add only new keys onto our list of keys to search
-				List<DeterministicKey> newKeys = allLeafKeys.subList(ki, allLeafKeys.size());
-				keys.addAll(newKeys);
-				// Fall-through to checking more keys as now 'ki' is smaller than 'keys.size()' again
 
 			// Process new keys
-			}
+		} while (true);
-
-			// If we have processed all keys, then we're done
-		} while (ki < keys.size());
-
-		return walletTransactions;
 	}
 
 	/**
 	 * Returns first unused receive address given 'm' BIP32 key.
 	 *
-	 * @param xprv58 BIP32 extended Bitcoin private key
+	 * @param key58 BIP32/HD extended Bitcoin private/public key
-	 * @return Bitcoin P2PKH address
+	 * @return P2PKH address
 	 * @throws ForeignBlockchainException if something went wrong
 	 */
-	public String getUnusedReceiveAddress(String xprv58) throws ForeignBlockchainException {
+	public String getUnusedReceiveAddress(String key58) throws ForeignBlockchainException {
 		Context.propagate(bitcoinjContext);
 
-		Wallet wallet = Wallet.fromSpendingKeyB58(this.params, xprv58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
+		Wallet wallet = walletFromDeterministicKey58(key58);
 		DeterministicKeyChain keyChain = wallet.getActiveKeyChain();
 
-		keyChain.setLookaheadSize(WalletAwareUTXOProvider.LOOKAHEAD_INCREMENT);
+		keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
 		keyChain.maybeLookAhead();
 
 		final int keyChainPathSize = keyChain.getAccountPath().size();
@@ -418,7 +400,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 				DeterministicKey dKey = keys.get(ki);
 				List<ChildNumber> dKeyPath = dKey.getPath();
 
-				// If keyChain is based on 'm', then make sure dKey is m/0/ki
+				// If keyChain is based on 'm', then make sure dKey is m/0/ki - i.e. a 'receive' address, not 'change' (m/1/ki)
 				if (dKeyPath.size() != keyChainPathSize + 2 || dKeyPath.get(dKeyPath.size() - 2) != ChildNumber.ZERO)
 					continue;
 
@@ -439,7 +421,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 				if (unspentOutputs.isEmpty()) {
 					// If this is a known key that has been spent before, then we can skip asking for transaction history
 					if (this.spentKeys.contains(dKey)) {
-						wallet.getActiveKeyChain().markKeyAsUsed((DeterministicKey) dKey);
+						wallet.getActiveKeyChain().markKeyAsUsed(dKey);
 						continue;
 					}
 
@@ -450,26 +432,19 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 						// Fully spent key - case (a)
 						this.spentKeys.add(dKey);
 						wallet.getActiveKeyChain().markKeyAsUsed(dKey);
-					} else {
+						continue;
+					}
+
 					// Key never been used - case (b)
 					return address.toString();
 				}
-				}
 
 				// Key has unspent outputs, hence used, so no good to us
 				this.spentKeys.remove(dKey);
 			}
 
 			// Generate some more keys
-			keyChain.setLookaheadSize(keyChain.getLookaheadSize() + WalletAwareUTXOProvider.LOOKAHEAD_INCREMENT);
+			keys.addAll(generateMoreKeys(keyChain));
-			keyChain.maybeLookAhead();
-
-			// This returns all keys, including those already in 'keys'
-			List<DeterministicKey> allLeafKeys = keyChain.getLeafKeys();
-			// Add only new keys onto our list of keys to search
-			List<DeterministicKey> newKeys = allLeafKeys.subList(ki, allLeafKeys.size());
-			keys.addAll(newKeys);
-			// Fall-through to checking more keys as now 'ki' is smaller than 'keys.size()' again
 
 			// Process new keys
 		} while (true);
@@ -478,8 +453,6 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 	// UTXOProvider support
 
 	static class WalletAwareUTXOProvider implements UTXOProvider {
-		private static final int LOOKAHEAD_INCREMENT = 3;
-
 		private final Bitcoiny bitcoiny;
 		private final Wallet wallet;
 
@@ -491,7 +464,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 			this.keyChain = this.wallet.getActiveKeyChain();
 
 			// Set up wallet's key chain
-			this.keyChain.setLookaheadSize(LOOKAHEAD_INCREMENT);
+			this.keyChain.setLookaheadSize(Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
 			this.keyChain.maybeLookAhead();
 		}
 
@@ -575,23 +548,15 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 					}
 				}
 
-				if (!areAllKeysUnspent) {
+				if (areAllKeysUnspent)
-					// Generate some more keys
+					// No transactions for this batch of keys so assume we're done searching.
-					this.keyChain.setLookaheadSize(this.keyChain.getLookaheadSize() + LOOKAHEAD_INCREMENT);
-					this.keyChain.maybeLookAhead();
-
-					// This returns all keys, including those already in 'keys'
-					List<DeterministicKey> allLeafKeys = this.keyChain.getLeafKeys();
-					// Add only new keys onto our list of keys to search
-					List<DeterministicKey> newKeys = allLeafKeys.subList(ki, allLeafKeys.size());
-					keys.addAll(newKeys);
-					// Fall-through to checking more keys as now 'ki' is smaller than 'keys.size()' again
-				}
-
-				// If we have processed all keys, then we're done
-			} while (ki < keys.size());
-
 					return allUnspentOutputs;
+
+				// Generate some more keys
+				keys.addAll(Bitcoiny.generateMoreKeys(this.keyChain));
+
+				// Process new keys
+			} while (true);
 		}
 
 		@Override
@@ -611,10 +576,34 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 
 	// Utility methods for us
 
+	protected static List<DeterministicKey> generateMoreKeys(DeterministicKeyChain keyChain) {
+		int existingLeafKeyCount = keyChain.getLeafKeys().size();
+
+		// Increase lookahead size so that...
+		keyChain.setLookaheadSize(keyChain.getLookaheadSize() + Bitcoiny.WALLET_KEY_LOOKAHEAD_INCREMENT);
+		// ...this call will generate more keys
+		keyChain.maybeLookAhead();
+
+		// This returns *all* keys
+		List<DeterministicKey> allLeafKeys = keyChain.getLeafKeys();
+
+		// Only return newly generated keys
+		return allLeafKeys.subList(existingLeafKeyCount, allLeafKeys.size());
+	}
+
 	protected byte[] addressToScriptPubKey(String base58Address) {
-		Context.propagate(bitcoinjContext);
+		Context.propagate(this.bitcoinjContext);
 		Address address = Address.fromString(this.params, base58Address);
 		return ScriptBuilder.createOutputScript(address).getProgram();
 	}
 
+	protected Wallet walletFromDeterministicKey58(String key58) {
+		DeterministicKey dKey = DeterministicKey.deserializeB58(null, key58, this.params);
+
+		if (dKey.hasPrivKey())
+			return Wallet.fromSpendingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
+		else
+			return Wallet.fromWatchingKeyB58(this.params, key58, DeterministicHierarchy.BIP32_STANDARDISATION_TIME_SECS);
+	}
+
 }

src/test/java/org/qortal/test/crosschain/apps/BuildHTLC.java

@@ -85,6 +85,8 @@ public class BuildHTLC {
 			usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny));
 		if (p2shFee.isZero())
 			return;

src/test/java/org/qortal/test/crosschain/apps/CheckHTLC.java

@@ -90,6 +90,8 @@ public class CheckHTLC {
 			usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny));
 		if (p2shFee.isZero())
 			return;

src/test/java/org/qortal/test/crosschain/apps/GetNextReceiveAddress.java

@@ -0,0 +1,78 @@
+package org.qortal.test.crosschain.apps;
+
+import java.security.Security;
+
+import org.bitcoinj.core.AddressFormatException;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
+import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.qortal.crosschain.Bitcoin;
+import org.qortal.crosschain.Bitcoiny;
+import org.qortal.crosschain.ForeignBlockchainException;
+import org.qortal.crosschain.Litecoin;
+import org.qortal.settings.Settings;
+
+public class GetNextReceiveAddress {
+
+	static {
+		// This must go before any calls to LogManager/Logger
+		System.setProperty("java.util.logging.manager", "org.apache.logging.log4j.jul.LogManager");
+	}
+
+	private static void usage(String error) {
+		if (error != null)
+			System.err.println(error);
+
+		System.err.println(String.format("usage: GetNextReceiveAddress (-b | -l) <xprv/xpub>"));
+		System.err.println(String.format("example (testnet): GetNextReceiveAddress -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB"));
+		System.exit(1);
+	}
+
+	public static void main(String[] args) {
+		if (args.length != 2)
+			usage(null);
+
+		Security.insertProviderAt(new BouncyCastleProvider(), 0);
+		Security.insertProviderAt(new BouncyCastleJsseProvider(), 1);
+
+		Settings.fileInstance("settings-test.json");
+
+		Bitcoiny bitcoiny = null;
+		String key58 = null;
+
+		int argIndex = 0;
+		try {
+			switch (args[argIndex++]) {
+				case "-b":
+					bitcoiny = Bitcoin.getInstance();
+					break;
+
+				case "-l":
+					bitcoiny = Litecoin.getInstance();
+					break;
+
+				default:
+					usage("Only Bitcoin (-b) or Litecoin (-l) supported");
+			}
+
+			key58 = args[argIndex++];
+
+			if (!bitcoiny.isValidDeterministicKey(key58))
+				usage("Not valid xprv/xpub/tprv/tpub");
+		} catch (NumberFormatException | AddressFormatException e) {
+			usage(String.format("Argument format exception: %s", e.getMessage()));
+		}
+
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
+		String receiveAddress = null;
+		try {
+			receiveAddress = bitcoiny.getUnusedReceiveAddress(key58);
+		} catch (ForeignBlockchainException e) {
+			System.err.println(String.format("Failed to determine next receive address: %s", e.getMessage()));
+			System.exit(1);
+		}
+
+		System.out.println(String.format("Next receive address: %s", receiveAddress));
+	}
+
+}

src/test/java/org/qortal/test/crosschain/apps/GetTransaction.java

@@ -1,4 +1,4 @@
-package org.qortal.test.crosschain;
+package org.qortal.test.crosschain.apps;
 
 import java.security.Security;
 import java.util.List;
@@ -8,7 +8,9 @@ import org.bitcoinj.core.TransactionOutput;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
 import org.qortal.crosschain.Bitcoin;
+import org.qortal.crosschain.Bitcoiny;
 import org.qortal.crosschain.ForeignBlockchainException;
+import org.qortal.crosschain.Litecoin;
 import org.qortal.settings.Settings;
 
 import com.google.common.hash.HashCode;
@@ -24,14 +26,14 @@ public class GetTransaction {
 		if (error != null)
 			System.err.println(error);
 
-		System.err.println(String.format("usage: GetTransaction <bitcoin-tx>"));
+		System.err.println(String.format("usage: GetTransaction (-b | -l) <tx-hash>"));
-		System.err.println(String.format("example (mainnet): GetTransaction 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660"));
+		System.err.println(String.format("example (mainnet): GetTransaction -b 816407e79568f165f13e09e9912c5f2243e0a23a007cec425acedc2e89284660"));
-		System.err.println(String.format("example (testnet): GetTransaction 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e"));
+		System.err.println(String.format("example (testnet): GetTransaction -b 3bfd17a492a4e3d6cb7204e17e20aca6c1ab82e1828bd1106eefbaf086fb8a4e"));
 		System.exit(1);
 	}
 
 	public static void main(String[] args) {
-		if (args.length != 1)
+		if (args.length != 2)
 			usage(null);
 
 		Security.insertProviderAt(new BouncyCastleProvider(), 0);
@@ -39,20 +41,35 @@ public class GetTransaction {
 
 		Settings.fileInstance("settings-test.json");
 
+		Bitcoiny bitcoiny = null;
 		byte[] transactionId = null;
 
-		try {
 		int argIndex = 0;
+		try {
+			switch (args[argIndex++]) {
+				case "-b":
+					bitcoiny = Bitcoin.getInstance();
+					break;
+
+				case "-l":
+					bitcoiny = Litecoin.getInstance();
+					break;
+
+				default:
+					usage("Only Bitcoin (-b) or Litecoin (-l) supported");
+			}
 
 			transactionId = HashCode.fromString(args[argIndex++]).asBytes();
 		} catch (NumberFormatException | AddressFormatException e) {
 			usage(String.format("Argument format exception: %s", e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		// Grab all outputs from transaction
 		List<TransactionOutput> fundingOutputs;
 		try {
-			fundingOutputs = Bitcoin.getInstance().getOutputs(transactionId);
+			fundingOutputs = bitcoiny.getOutputs(transactionId);
 		} catch (ForeignBlockchainException e) {
 			System.out.println(String.format("Transaction not found (or error occurred)"));
 			return;

src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java

@@ -1,4 +1,4 @@
-package org.qortal.test.crosschain;
+package org.qortal.test.crosschain.apps;
 
 import java.security.Security;
 import java.util.Comparator;
@@ -8,6 +8,7 @@ import java.util.stream.Collectors;
 import org.bitcoinj.core.AddressFormatException;
 import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.bouncycastle.jsse.provider.BouncyCastleJsseProvider;
+import org.qortal.crosschain.Bitcoin;
 import org.qortal.crosschain.Bitcoiny;
 import org.qortal.crosschain.BitcoinyTransaction;
 import org.qortal.crosschain.ForeignBlockchainException;
@@ -25,13 +26,13 @@ public class GetWalletTransactions {
 		if (error != null)
 			System.err.println(error);
 
-		System.err.println(String.format("usage: GetWalletTransactions <xprv/xpub>"));
+		System.err.println(String.format("usage: GetWalletTransactions (-b | -l) <xprv/xpub>"));
-		System.err.println(String.format("example (testnet): GetWalletTransactions tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB"));
+		System.err.println(String.format("example (testnet): GetWalletTransactions -l tpubD6NzVbkrYhZ4X3jV96Wo3Kr8Au2v9cUUEmPRk1smwduFrRVfBjkkw49rRYjgff1fGSktFMfabbvv8b1dmfyLjjbDax6QGyxpsNsx5PXukCB"));
 		System.exit(1);
 	}
 
 	public static void main(String[] args) {
-		if (args.length != 1)
+		if (args.length != 2)
 			usage(null);
 
 		Security.insertProviderAt(new BouncyCastleProvider(), 0);
@@ -39,12 +40,23 @@ public class GetWalletTransactions {
 
 		Settings.fileInstance("settings-test.json");
 
-		Bitcoiny bitcoiny = Litecoin.getInstance();
+		Bitcoiny bitcoiny = null;
-
 		String key58 = null;
 
-		try {
 		int argIndex = 0;
+		try {
+			switch (args[argIndex++]) {
+				case "-b":
+					bitcoiny = Bitcoin.getInstance();
+					break;
+
+				case "-l":
+					bitcoiny = Litecoin.getInstance();
+					break;
+
+				default:
+					usage("Only Bitcoin (-b) or Litecoin (-l) supported");
+			}
 
 			key58 = args[argIndex++];
 
@@ -54,6 +66,8 @@ public class GetWalletTransactions {
 			usage(String.format("Argument format exception: %s", e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		// Grab all outputs from transaction
 		Set<BitcoinyTransaction> transactions = null;
 		try {

src/test/java/org/qortal/test/crosschain/apps/Pay.java

@@ -63,6 +63,8 @@ public class Pay {
 			usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		System.out.println(String.format("Address: %s", address));
 		System.out.println(String.format("Amount: %s", amount.toPlainString()));
 

src/test/java/org/qortal/test/crosschain/apps/RedeemHTLC.java

@@ -102,6 +102,8 @@ public class RedeemHTLC {
 			usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny));
 		if (p2shFee.isZero())
 			return;

src/test/java/org/qortal/test/crosschain/apps/RefundHTLC.java

@@ -102,6 +102,8 @@ public class RefundHTLC {
 			usage(String.format("Invalid argument %d: %s", argIndex, e.getMessage()));
 		}
 
+		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
+
 		Coin p2shFee = Coin.valueOf(Common.getP2shFee(bitcoiny));
 		if (p2shFee.isZero())
 			return;