Improvements to ElectrumX, Bitcoin-y aspects, etc.

Add caching of transactions fetched via ElectrumX to reduce network load and speed up API response.

Fix handling ElectrumX servers that don't want to supply verbose transaction JSON.

Hide lots of data in BitcoinyTransaction that isn't needed by current API users.

src/main/java/org/qortal/api/model/SimpleForeignTransaction.java

@@ -0,0 +1,157 @@
+package org.qortal.api.model;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+
+@XmlAccessorType(XmlAccessType.FIELD)
+public class SimpleForeignTransaction {
+
+	public static class AddressAmount {
+		public final String address;
+		public final long amount;
+
+		protected AddressAmount() {
+			/* For JAXB */
+			this.address = null;
+			this.amount = 0;
+		}
+
+		public AddressAmount(String address, long amount) {
+			this.address = address;
+			this.amount = amount;
+		}
+	}
+
+	private String txHash;
+	private long timestamp;
+
+	private List<AddressAmount> inputs;
+
+	public static class Output {
+		public final List<String> addresses;
+		public final long amount;
+
+		protected Output() {
+			/* For JAXB */
+			this.addresses = null;
+			this.amount = 0;
+		}
+
+		public Output(List<String> addresses, long amount) {
+			this.addresses = addresses;
+			this.amount = amount;
+		}
+	}
+	private List<Output> outputs;
+
+	private long totalAmount;
+	private long fees;
+
+	private Boolean isSentNotReceived;
+
+	protected SimpleForeignTransaction() {
+		/* For JAXB */
+	}
+
+	private SimpleForeignTransaction(Builder builder) {
+		this.txHash = builder.txHash;
+		this.timestamp = builder.timestamp;
+		this.inputs = Collections.unmodifiableList(builder.inputs);
+		this.outputs = Collections.unmodifiableList(builder.outputs);
+
+		Objects.requireNonNull(this.txHash);
+		if (timestamp <= 0)
+			throw new IllegalArgumentException("timestamp must be positive");
+
+		long totalGrossAmount = this.inputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum);
+		this.totalAmount = this.outputs.stream().map(addressAmount -> addressAmount.amount).reduce(0L, Long::sum);
+
+		this.fees = totalGrossAmount - this.totalAmount;
+
+		this.isSentNotReceived = builder.isSentNotReceived;
+	}
+
+	public String getTxHash() {
+		return this.txHash;
+	}
+
+	public long getTimestamp() {
+		return this.timestamp;
+	}
+
+	public List<AddressAmount> getInputs() {
+		return this.inputs;
+	}
+
+	public List<Output> getOutputs() {
+		return this.outputs;
+	}
+
+	public long getTotalAmount() {
+		return this.totalAmount;
+	}
+
+	public long getFees() {
+		return this.fees;
+	}
+
+	public Boolean isSentNotReceived() {
+		return this.isSentNotReceived;
+	}
+
+	public static class Builder {
+		private String txHash;
+		private long timestamp;
+		private List<AddressAmount> inputs = new ArrayList<>();
+		private List<Output> outputs = new ArrayList<>();
+		private Boolean isSentNotReceived;
+
+		public Builder txHash(String txHash) {
+			this.txHash = Objects.requireNonNull(txHash);
+			return this;
+		}
+
+		public Builder timestamp(long timestamp) {
+			if (timestamp <= 0)
+				throw new IllegalArgumentException("timestamp must be positive");
+
+			this.timestamp = timestamp;
+			return this;
+		}
+
+		public Builder input(String address, long amount) {
+			Objects.requireNonNull(address);
+			if (amount < 0)
+				throw new IllegalArgumentException("amount must be zero or positive");
+
+			AddressAmount input = new AddressAmount(address, amount);
+			inputs.add(input);
+			return this;
+		}
+
+		public Builder output(List<String> addresses, long amount) {
+			Objects.requireNonNull(addresses);
+			if (amount < 0)
+				throw new IllegalArgumentException("amount must be zero or positive");
+
+			Output output = new Output(addresses, amount);
+			outputs.add(output);
+			return this;
+		}
+
+		public Builder isSentNotReceived(Boolean isSentNotReceived) {
+			this.isSentNotReceived = isSentNotReceived;
+			return this;
+		}
+
+		public SimpleForeignTransaction build() {
+			return new SimpleForeignTransaction(this);
+		}
+	}
+
+}

src/main/java/org/qortal/crosschain/Bitcoiny.java

@@ -2,8 +2,10 @@ package org.qortal.crosschain;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -31,6 +33,7 @@ import org.bitcoinj.script.ScriptBuilder;
 import org.bitcoinj.wallet.DeterministicKeyChain;
 import org.bitcoinj.wallet.SendRequest;
 import org.bitcoinj.wallet.Wallet;
+import org.qortal.api.model.SimpleForeignTransaction;
 import org.qortal.crypto.Crypto;
 import org.qortal.utils.Amounts;
 import org.qortal.utils.BitTwiddling;
@@ -329,7 +332,7 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		return balance.value;
 	}
 
-	public Set<BitcoinyTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
+	public List<BitcoinyTransaction> getWalletTransactions(String key58) throws ForeignBlockchainException {
 		Context.propagate(bitcoinjContext);
 
 		Wallet wallet = walletFromDeterministicKey58(key58);
@@ -366,13 +369,15 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 
 			if (areAllKeysUnused)
 				// No transactions for this batch of keys so assume we're done searching.
-				return walletTransactions;
+				break;
 
 			// Generate some more keys
 			keys.addAll(generateMoreKeys(keyChain));
 
 			// Process new keys
 		} while (true);
+
+		return walletTransactions.stream().collect(Collectors.toList());
 	}
 
 	/**
@@ -574,6 +579,94 @@ public abstract class Bitcoiny implements ForeignBlockchain {
 		}
 	}
 
+	// Utility methods for others
+
+	public static List<SimpleForeignTransaction> simplifyWalletTransactions(List<BitcoinyTransaction> transactions) {
+		// Sort by oldest timestamp first
+		transactions.sort(Comparator.comparingInt(t -> t.timestamp));
+
+		// Manual 2nd-level sort same-timestamp transactions so that a transaction's input comes first
+		int fromIndex = 0;
+		do {
+			int timestamp = transactions.get(fromIndex).timestamp;
+
+			int toIndex;
+			for (toIndex = fromIndex + 1; toIndex < transactions.size(); ++toIndex)
+				if (transactions.get(toIndex).timestamp != timestamp)
+					break;
+
+			// Process same-timestamp sub-list
+			List<BitcoinyTransaction> subList = transactions.subList(fromIndex, toIndex);
+
+			// Only if necessary
+			if (subList.size() > 1) {
+				// Quick index lookup
+				Map<String, Integer> indexByTxHash = subList.stream().collect(Collectors.toMap(t -> t.txHash, t -> t.timestamp));
+
+				int restartIndex = 0;
+				boolean isSorted;
+				do {
+					isSorted = true;
+
+					for (int ourIndex = restartIndex; ourIndex < subList.size(); ++ourIndex) {
+						BitcoinyTransaction ourTx = subList.get(ourIndex);
+
+						for (BitcoinyTransaction.Input input : ourTx.inputs) {
+							Integer inputIndex = indexByTxHash.get(input.outputTxHash);
+
+							if (inputIndex != null && inputIndex > ourIndex) {
+								// Input tx is currently after current tx, so swap
+								BitcoinyTransaction tmpTx = subList.get(inputIndex);
+								subList.set(inputIndex, ourTx);
+								subList.set(ourIndex, tmpTx);
+
+								// Update index lookup too
+								indexByTxHash.put(ourTx.txHash, inputIndex);
+								indexByTxHash.put(tmpTx.txHash, ourIndex);
+
+								if (isSorted)
+									restartIndex = Math.max(restartIndex, ourIndex);
+
+								isSorted = false;
+								break;
+							}
+						}
+					}
+				} while (!isSorted);
+			}
+
+			fromIndex = toIndex;
+		} while (fromIndex < transactions.size());
+
+		// Simplify
+		List<SimpleForeignTransaction> simpleTransactions = new ArrayList<>();
+
+		// Quick lookup of txs in our wallet
+		Set<String> walletTxHashes = transactions.stream().map(t -> t.txHash).collect(Collectors.toSet());
+
+		for (BitcoinyTransaction transaction : transactions) {
+			SimpleForeignTransaction.Builder builder = new SimpleForeignTransaction.Builder();
+			builder.txHash(transaction.txHash);
+			builder.timestamp(transaction.timestamp);
+
+			builder.isSentNotReceived(false);
+
+			for (BitcoinyTransaction.Input input : transaction.inputs) {
+				// TODO: add input via builder
+
+				if (walletTxHashes.contains(input.outputTxHash))
+					builder.isSentNotReceived(true);
+			}
+
+			for (BitcoinyTransaction.Output output : transaction.outputs)
+				builder.output(output.addresses, output.value);
+
+			simpleTransactions.add(builder.build());
+		}
+
+		return simpleTransactions;
+	}
+
 	// Utility methods for us
 
 	protected static List<DeterministicKey> generateMoreKeys(DeterministicKeyChain keyChain) {

src/main/java/org/qortal/crosschain/BitcoinyBlockchainProvider.java

@@ -19,6 +19,9 @@ public abstract class BitcoinyBlockchainProvider {
 	/** Returns balance of address represented by <tt>scriptPubKey</tt>. */
 	public abstract long getConfirmedBalance(byte[] scriptPubKey) throws ForeignBlockchainException;
 
+	/** Returns raw, serialized, transaction bytes given <tt>txHash</tt>. */
+	public abstract byte[] getRawTransaction(String txHash) throws ForeignBlockchainException;
+
 	/** Returns raw, serialized, transaction bytes given <tt>txHash</tt>. */
 	public abstract byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException;
 

src/main/java/org/qortal/crosschain/BitcoinyTransaction.java

@@ -1,25 +1,35 @@
 package org.qortal.crosschain;
 
 import java.util.List;
-import java.util.Set;
 import java.util.stream.Collectors;
 
 import javax.xml.bind.annotation.XmlAccessType;
 import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlTransient;
 
 @XmlAccessorType(XmlAccessType.FIELD)
 public class BitcoinyTransaction {
 
 	public final String txHash;
+
+	@XmlTransient
 	public final int size;
+
+	@XmlTransient
 	public final int locktime;
+
 	// Not present if transaction is unconfirmed
 	public final Integer timestamp;
 
 	public static class Input {
+		@XmlTransient
 		public final String scriptSig;
+
+		@XmlTransient
 		public final int sequence;
+
 		public final String outputTxHash;
+
 		public final int outputVout;
 
 		// For JAXB
@@ -42,12 +52,16 @@ public class BitcoinyTransaction {
 					this.outputTxHash, this.outputVout, this.sequence, this.scriptSig);
 		}
 	}
+	@XmlTransient
 	public final List<Input> inputs;
 
 	public static class Output {
+		@XmlTransient
 		public final String scriptPubKey;
+
 		public final long value;
-		public final Set<String> addresses;
+
+		public final List<String> addresses;
 
 		// For JAXB
 		protected Output() {
@@ -62,7 +76,7 @@ public class BitcoinyTransaction {
 			this.addresses = null;
 		}
 
-		public Output(String scriptPubKey, long value, Set<String> addresses) {
+		public Output(String scriptPubKey, long value, List<String> addresses) {
 			this.scriptPubKey = scriptPubKey;
 			this.value = value;
 			this.addresses = addresses;
@@ -74,6 +88,8 @@ public class BitcoinyTransaction {
 	}
 	public final List<Output> outputs;
 
+	public final long totalAmount;
+
 	// For JAXB
 	protected BitcoinyTransaction() {
 		this.txHash = null;
@@ -82,6 +98,7 @@ public class BitcoinyTransaction {
 		this.timestamp = 0;
 		this.inputs = null;
 		this.outputs = null;
+		this.totalAmount = 0;
 	}
 
 	public BitcoinyTransaction(String txHash, int size, int locktime, Integer timestamp,
@@ -92,6 +109,8 @@ public class BitcoinyTransaction {
 		this.timestamp = timestamp;
 		this.inputs = inputs;
 		this.outputs = outputs;
+
+		this.totalAmount = outputs.stream().map(output -> output.value).reduce(0L, Long::sum);
 	}
 
 	public String toString() {

src/main/java/org/qortal/crosschain/ElectrumX.java

@@ -10,6 +10,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumMap;
 import java.util.HashSet;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -100,6 +101,16 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	private Scanner scanner;
 	private int nextId = 1;
 
+	private static final int TX_CACHE_SIZE = 100;
+	@SuppressWarnings("serial")
+	private final Map<String, BitcoinyTransaction> transactionCache = Collections.synchronizedMap(new LinkedHashMap<>(TX_CACHE_SIZE + 1, 0.75F, true) {
+		// This method is called just after a new entry has been added
+		@Override
+		public boolean removeEldestEntry(Map.Entry<String, BitcoinyTransaction> eldest) {
+			return size() > TX_CACHE_SIZE;
+		}
+	});
+
 	// Constructors
 
 	public ElectrumX(String netId, String genesisHash, Collection<Server> initialServerList, Map<Server.ConnectionType, Integer> defaultPorts) {
@@ -232,14 +243,16 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	/**
 	 * Returns raw transaction for passed transaction hash.
 	 * <p>
+	 * NOTE: Do not mutate returned byte[]!
+	 * 
 	 * @throws ForeignBlockchainException.NotFoundException if transaction not found
 	 * @throws ForeignBlockchainException if error occurs
 	 */
 	@Override
-	public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException {
+	public byte[] getRawTransaction(String txHash) throws ForeignBlockchainException {
 		Object rawTransactionHex;
 		try {
-			rawTransactionHex = this.rpc("blockchain.transaction.get", HashCode.fromBytes(txHash).toString(), false);
+			rawTransactionHex = this.rpc("blockchain.transaction.get", txHash, false);
 		} catch (ForeignBlockchainException.NetworkException e) {
 			// DaemonError({'code': -5, 'message': 'No such mempool or blockchain transaction. Use gettransaction for wallet transactions.'})
 			if (Integer.valueOf(-5).equals(e.getDaemonErrorCode()))
@@ -254,6 +267,19 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 		return HashCode.fromString((String) rawTransactionHex).asBytes();
 	}
 
+	/**
+	 * Returns raw transaction for passed transaction hash.
+	 * <p>
+	 * NOTE: Do not mutate returned byte[]!
+	 * 
+	 * @throws ForeignBlockchainException.NotFoundException if transaction not found
+	 * @throws ForeignBlockchainException if error occurs
+	 */
+	@Override
+	public byte[] getRawTransaction(byte[] txHash) throws ForeignBlockchainException {
+		return getRawTransaction(HashCode.fromBytes(txHash).toString());
+	}
+
 	/**
 	 * Returns transaction info for passed transaction hash.
 	 * <p>
@@ -262,6 +288,11 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 	 */
 	@Override
 	public BitcoinyTransaction getTransaction(String txHash) throws ForeignBlockchainException {
+		// Check cache first
+		BitcoinyTransaction transaction = transactionCache.get(txHash);
+		if (transaction != null)
+			return transaction;
+
 		Object transactionObj = null;
 
 		do {
@@ -275,7 +306,7 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 				// Some servers also return non-standard responses like this:
 				// {"error":"verbose transactions are currently unsupported","id":3,"jsonrpc":"2.0"}
 				// We should probably not use this server any more
-				if (e.getServer() != null && VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE.equals(e.getMessage())) {
+				if (e.getServer() != null && e.getMessage() != null && e.getMessage().contains(VERBOSE_TRANSACTIONS_UNSUPPORTED_MESSAGE)) {
 					Server uselessServer = (Server) e.getServer();
 					LOGGER.trace(() -> String.format("Server %s doesn't support verbose transactions - barring use of that server", uselessServer));
 					this.uselessServers.add(uselessServer);
@@ -330,10 +361,10 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 				long value = (long) (((Double) outputJson.get("value")) * 1e8);
 
 				// address too, if present
-				Set<String> addresses = null;
+				List<String> addresses = null;
 				Object addressesObj = ((JSONObject) outputJson.get("scriptPubKey")).get("addresses");
 				if (addressesObj instanceof JSONArray) {
-					addresses = new HashSet<>();
+					addresses = new ArrayList<>();
 					for (Object addressObj : (JSONArray) addressesObj)
 						addresses.add((String) addressObj);
 				}
@@ -341,7 +372,12 @@ public class ElectrumX extends BitcoinyBlockchainProvider {
 				outputs.add(new BitcoinyTransaction.Output(scriptPubKey, value, addresses));
 			}
 
-			return new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs);
+			transaction = new BitcoinyTransaction(txHash, size, locktime, timestamp, inputs, outputs);
+
+			// Save into cache
+			transactionCache.put(txHash, transaction);
+
+			return transaction;
 		} catch (NullPointerException | ClassCastException e) {
 			// Unexpected / invalid response from ElectrumX server
 		}

src/test/java/org/qortal/test/crosschain/apps/GetWalletTransactions.java

@@ -2,7 +2,7 @@ package org.qortal.test.crosschain.apps;
 
 import java.security.Security;
 import java.util.Comparator;
-import java.util.Set;
+import java.util.List;
 import java.util.stream.Collectors;
 
 import org.bitcoinj.core.AddressFormatException;
@@ -69,7 +69,7 @@ public class GetWalletTransactions {
 		System.out.println(String.format("Using %s", bitcoiny.getBlockchainProvider().getNetId()));
 
 		// Grab all outputs from transaction
-		Set<BitcoinyTransaction> transactions = null;
+		List<BitcoinyTransaction> transactions = null;
 		try {
 			transactions = bitcoiny.getWalletTransactions(key58);
 		} catch (ForeignBlockchainException e) {