diff --git a/rust-lightclient/proto/service.proto b/rust-lightclient/proto/service.proto index 0b797e3..d6cf040 100644 --- a/rust-lightclient/proto/service.proto +++ b/rust-lightclient/proto/service.proto @@ -52,6 +52,11 @@ message TransparentAddress { string address = 1; } +message TransparentAddressBlockFilter { + string address = 1; + BlockRange range = 2; +} + message Utxo { TransparentAddress address = 1; bytes txid = 2; @@ -73,6 +78,7 @@ service CompactTxStreamer { // t-Address support rpc GetUtxos(TransparentAddress) returns (stream Utxo) {} + rpc GetAddressTxids(TransparentAddressBlockFilter) returns (stream RawTransaction) {} // Misc rpc GetLightdInfo(Empty) returns (LightdInfo) {} diff --git a/rust-lightclient/src/lightclient.rs b/rust-lightclient/src/lightclient.rs index c815918..cb72ae4 100644 --- a/rust-lightclient/src/lightclient.rs +++ b/rust-lightclient/src/lightclient.rs @@ -26,7 +26,8 @@ use tower_hyper::{client, util}; use tower_util::MakeService; use futures::stream::Stream; -use crate::grpc_client::{ChainSpec, BlockId, BlockRange, RawTransaction, TransparentAddress, TxFilter, Empty}; +use crate::grpc_client::{ChainSpec, BlockId, BlockRange, RawTransaction, TransparentAddress, + TransparentAddressBlockFilter, TxFilter, Empty}; use crate::grpc_client::client::CompactTxStreamer; // Used below to return the grpc "Client" type to calling methods @@ -320,16 +321,29 @@ impl LightClient { let local_light_wallet = self.wallet.clone(); let local_bytes_downloaded = bytes_downloaded.clone(); - let simple_callback = move |encoded_block: &[u8]| { - local_light_wallet.scan_block(encoded_block); - local_bytes_downloaded.fetch_add(encoded_block.len(), Ordering::SeqCst); - }; - print!("Syncing {}/{}, Balance = {} \r", last_scanned_height, last_block, self.wallet.balance(None)); - self.fetch_blocks(last_scanned_height, end_height, simple_callback); + // Fetch compact blocks + self.fetch_blocks(last_scanned_height, end_height, + move |encoded_block: &[u8]| { + local_light_wallet.scan_block(encoded_block); + local_bytes_downloaded.fetch_add(encoded_block.len(), Ordering::SeqCst); + }); + // We'll also fetch all the txids that our transparent addresses are involved with + // TODO: Use for all t addresses + let address = LightWallet::address_from_sk(&self.wallet.tkeys[0]); + let wallet = self.wallet.clone(); + self.fetch_transparent_txids(address, last_scanned_height, end_height, + move |tx_bytes: &[u8] | { + let tx = Transaction::read(tx_bytes).unwrap(); + + // Scan this Tx for transparent inputs and outputs + wallet.scan_full_tx(&tx, -1); // TODO: Add the height here! + } + ); + last_scanned_height = end_height + 1; end_height = last_scanned_height + 1000 - 1; @@ -348,7 +362,7 @@ impl LightClient { // We need to first copy over the Txids from the wallet struct, because // we need to free the read lock from here (Because we'll self.wallet.txs later) - let txids_to_fetch: Vec; + let txids_to_fetch: Vec<(TxId, i32)>; { // First, build a list of all the TxIDs and Memos that we need // to fetch. @@ -361,53 +375,45 @@ impl LightClient { .flat_map( |(txid, wtx)| { // flat_map because we're collecting vector of vectors wtx.notes.iter() .filter( |nd| nd.memo.is_none()) // only get if memo is None (i.e., it has not been fetched) - .map( |nd| (txid.clone(), nd.memo.clone()) ) // collect (txid, memo) Clone everything because we want copies, so we can release the read lock - .collect::)>>() // convert to vector + .map( |nd| (txid.clone(), nd.memo.clone(), wtx.block) ) // collect (txid, memo, height) Clone everything because we want copies, so we can release the read lock + .collect::, i32)>>() // convert to vector }) - .collect::)>>(); + .collect::, i32)>>(); //println!("{:?}", txids_and_memos); // TODO: Assert that all the memos here are None txids_to_fetch = txids_and_memos.iter() - .map( | (txid, _) | txid.clone() ) // We're only interested in the txids, so drop the Memo, which is None anyway - .collect::>(); // and convert into Vec + .map( | (txid, _, h) | (txid.clone(), *h) ) // We're only interested in the txids, so drop the Memo, which is None anyway + .collect::>(); // and convert into Vec } // And go and fetch the txids, getting the full transaction, so we can // read the memos - for txid in txids_to_fetch { + for (txid, height) in txids_to_fetch { let light_wallet_clone = self.wallet.clone(); println!("Fetching full Tx: {}", txid); self.fetch_full_tx(txid, move |tx_bytes: &[u8] | { let tx = Transaction::read(tx_bytes).unwrap(); - light_wallet_clone.scan_full_tx(&tx); + light_wallet_clone.scan_full_tx(&tx, height); }); }; // Finally, fetch the UTXOs - // Get all the UTXOs for our transparent addresses - // TODO: This is a super hack. Clear all UTXOs - { - let mut txs = self.wallet.txs.write().unwrap(); - for tx in txs.values_mut() { - tx.utxos.clear(); - } - } - + // Get all the UTXOs for our transparent addresses, clearing out the current list + self.wallet.clear_utxos(); // Fetch UTXOs self.wallet.tkeys.iter() .map( |sk| LightWallet::address_from_sk(&sk)) .for_each( |taddr| { let wallet = self.wallet.clone(); self.fetch_utxos(taddr, move |utxo| { - wallet.scan_utxo(&utxo); + wallet.add_utxo(&utxo); }); }); - } pub fn do_send(&self, addr: &str, value: u64, memo: Option) { @@ -536,6 +542,56 @@ impl LightClient { tokio::runtime::current_thread::Runtime::new().unwrap().block_on(say_hello).unwrap(); } + pub fn fetch_transparent_txids(&self, address: String, + start_height: u64, end_height: u64,c: F) + where F : Fn(&[u8]) { + let uri: http::Uri = format!("http://127.0.0.1:9067").parse().unwrap(); + + let dst = Destination::try_from_uri(uri.clone()).unwrap(); + let connector = util::Connector::new(HttpConnector::new(4)); + let settings = client::Builder::new().http2_only(true).clone(); + let mut make_client = client::Connect::with_builder(connector, settings); + + let say_hello = make_client + .make_service(dst) + .map_err(|e| panic!("connect error: {:?}", e)) + .and_then(move |conn| { + + let conn = tower_request_modifier::Builder::new() + .set_origin(uri) + .build(conn) + .unwrap(); + + // Wait until the client is ready... + CompactTxStreamer::new(conn) + .ready() + .map_err(|e| eprintln!("streaming error {:?}", e)) + }) + .and_then(move |mut client| { + let start = Some(BlockId{ height: start_height, hash: vec!()}); + let end = Some(BlockId{ height: end_height, hash: vec!()}); + + let br = Request::new(TransparentAddressBlockFilter{ address, range: Some(BlockRange{start, end}) }); + + client + .get_address_txids(br) + .map_err(|e| { + eprintln!("RouteChat request failed; err={:?}", e); + }) + .and_then(move |response| { + let inbound = response.into_inner(); + inbound.for_each(move |tx| { + //let tx = Transaction::read(&tx.into_inner().data[..]).unwrap(); + c(&tx.data); + + Ok(()) + }) + .map_err(|e| eprintln!("gRPC inbound stream error: {:?}", e)) + }) + }); + + tokio::runtime::current_thread::Runtime::new().unwrap().block_on(say_hello).unwrap(); + } pub fn fetch_full_tx(&self, txid: TxId, c: F) where F : Fn(&[u8]) { diff --git a/rust-lightclient/src/lightwallet.rs b/rust-lightclient/src/lightwallet.rs index 843f9f9..0d7d24b 100644 --- a/rust-lightclient/src/lightwallet.rs +++ b/rust-lightclient/src/lightwallet.rs @@ -28,7 +28,7 @@ use zcash_primitives::{ components::{Amount, OutPoint, TxOut}, components::amount::DEFAULT_FEE, TxId, Transaction, }, - legacy::{Script, TransparentAddress}, + legacy::{Script, TransparentAddress}, note_encryption::{Memo, try_sapling_note_decryption}, zip32::{ExtendedFullViewingKey, ExtendedSpendingKey, ChildIndex}, JUBJUB, @@ -339,9 +339,81 @@ pub struct Utxo { } impl Utxo { + pub fn serialized_version() -> u64 { + return 1; + } + fn to_outpoint(&self) -> OutPoint { OutPoint { hash: self.txid.0, n: self.output_index as u32 } } + + pub fn read(mut reader: R) -> io::Result { + let version = reader.read_u64::()?; + assert_eq!(version, Utxo::serialized_version()); + + let address_len = reader.read_i32::()?; + let mut address_bytes = vec![0; address_len as usize]; + reader.read_exact(&mut address_bytes)?; + let address = String::from_utf8(address_bytes).unwrap(); + assert_eq!(address.chars().take(1).collect::>()[0], 't'); + + let mut txid_bytes = [0; 32]; + reader.read_exact(&mut txid_bytes)?; + let txid = TxId { 0: txid_bytes }; + + let output_index = reader.read_u64::()?; + let value = reader.read_u64::()?; + let height = reader.read_i32::()?; + + let script = Vector::read(&mut reader, |r| { + let mut byte = [0; 1]; + r.read_exact(&mut byte)?; + Ok(byte[0]) + })?; + + let spent = Optional::read(&mut reader, |r| { + let mut txbytes = [0u8; 32]; + r.read_exact(&mut txbytes)?; + Ok(TxId{0: txbytes}) + })?; + + let unconfirmed_spent = Optional::read(&mut reader, |r| { + let mut txbytes = [0; 32]; + r.read_exact(&mut txbytes)?; + Ok(TxId{0: txbytes}) + })?; + + Ok(Utxo { + address, + txid, + output_index, + script, + value, + height, + spent, + unconfirmed_spent, + }) + } + + pub fn write(&self, mut writer: W) -> io::Result<()> { + writer.write_u64::(Utxo::serialized_version())?; + + writer.write_u32::(self.address.as_bytes().len() as u32)?; + writer.write_all(self.address.as_bytes())?; + + writer.write_all(&self.txid.0)?; + + writer.write_u64::(self.output_index)?; + writer.write_u64::(self.value)?; + writer.write_i32::(self.height)?; + + Vector::write(&mut writer, &self.script, |w, b| w.write_all(&[*b]))?; + + Optional::write(&mut writer, &self.spent, |w, txid| w.write_all(&txid.0))?; + Optional::write(&mut writer, &self.unconfirmed_spent, |w, txid| w.write_all(&txid.0))?; + + Ok(()) + } } pub struct WalletTx { @@ -462,9 +534,12 @@ pub struct LightWallet { extfvks: Vec, pub address: Vec>, - // Transparent keys. TODO: Make it not pub + // Transparent keys. TODO: Make it not pubic pub tkeys: Vec, + // Current UTXOs that can be spent + utxos: Arc>>, + blocks: Arc>>, pub txs: Arc>>, } @@ -518,6 +593,7 @@ impl LightWallet { extfvks: vec![extfvk], address: vec![address], tkeys: vec![tpk], + utxos: Arc::new(RwLock::new(vec![])), blocks: Arc::new(RwLock::new(vec![])), txs: Arc::new(RwLock::new(HashMap::new())), }) @@ -545,6 +621,8 @@ impl LightWallet { // TODO: Generate transparent addresses from the seed let tpk = secp256k1::SecretKey::from_slice(&[1u8; 32]).unwrap(); + let utxos = Vector::read(&mut reader, |r| Utxo::read(r))?; + let blocks = Vector::read(&mut reader, |r| BlockData::read(r))?; let txs_tuples = Vector::read(&mut reader, |r| { @@ -561,6 +639,7 @@ impl LightWallet { extfvks: extfvks, address: addresses, tkeys: vec![tpk], + utxos: Arc::new(RwLock::new(utxos)), blocks: Arc::new(RwLock::new(blocks)), txs: Arc::new(RwLock::new(txs)) }) @@ -578,6 +657,8 @@ impl LightWallet { |w, sk| sk.write(w) )?; + Vector::write(&mut writer, &self.utxos.read().unwrap(), |w, u| u.write(w))?; + Vector::write(&mut writer, &self.blocks.read().unwrap(), |w, b| b.write(w))?; // The hashmap, write as a set of tuples @@ -683,6 +764,15 @@ impl LightWallet { hash160.result().to_base58check(&B58_PUBKEY_ADDRESS_PREFIX, &[]) } + pub fn address_from_pubkeyhash(ta: Option) -> Option { + match ta { + Some(TransparentAddress::PublicKey(hash)) => { + Some(hash.to_base58check(&B58_PUBKEY_ADDRESS_PREFIX, &[])) + }, + _ => None + } + } + pub fn get_seed_phrase(&self) -> String { Mnemonic::from_entropy(&self.seed, Language::English, @@ -760,8 +850,44 @@ impl LightWallet { .sum::() } + fn add_toutput_to_wtx(&self, height: i32, txid: &TxId, vout: &TxOut, n: u64) { + let mut txs = self.txs.write().unwrap(); + + // Find the existing transaction entry, or create a new one. + if !txs.contains_key(&txid) { + let tx_entry = WalletTx::new(height, &txid); + txs.insert(txid.clone(), tx_entry); + } + let tx_entry = txs.get_mut(&txid).unwrap(); + + // Make sure the vout isn't already there. + match tx_entry.utxos.iter().find(|utxo| { + utxo.txid == *txid && utxo.output_index == n && Amount::from_u64(utxo.value).unwrap() == vout.value + }) { + Some(_) => { /* We already have the txid as an output, do nothing */} + None => { + let address = LightWallet::address_from_pubkeyhash(vout.script_pubkey.address()); + if address.is_none() { + println!("Couldn't determine address for output!"); + } + //println!("Added {}, {}", txid, n); + // Add the utxo + tx_entry.utxos.push(Utxo{ + address: address.unwrap(), + txid: txid.clone(), + output_index: n, + script: vout.script_pubkey.0.clone(), + value: vout.value.into(), + height: height, + spent: None, + unconfirmed_spent: None, + }); + } + } + } + // Scan the full Tx and update memos for incoming shielded transactions - pub fn scan_full_tx(&self, tx: &Transaction) { + pub fn scan_full_tx(&self, tx: &Transaction, height: i32) { // Scan all the inputs to see if we spent any transparent funds in this tx // TODO: Save this object @@ -773,15 +899,16 @@ impl LightWallet { let mut total_transparent_spend: u64 = 0; - println!("looking for t inputs"); for vin in tx.vin.iter() { - println!("looking for t inputs inside"); match vin.script_sig.public_key() { Some(pk) => { - println!("One of our transparent inputs was spent. {}, {}", hex::encode(pk.to_vec()), hex::encode(pubkey.to_vec())); + //println!("One of our transparent inputs was spent. {}, {}", hex::encode(pk.to_vec()), hex::encode(pubkey.to_vec())); if pk[..] == pubkey[..] { // Find the txid in the list of utxos that we have. let txid = TxId {0: vin.prevout.hash}; + + // println!("Looking for {}, {}", txid, vin.prevout.n); + let value = match self.txs.read().unwrap().get(&txid) { Some(wtx) => { // One of the tx outputs is a match @@ -803,10 +930,29 @@ impl LightWallet { }; } - { + // Scan for t outputs + for (n, vout) in tx.vout.iter().enumerate() { + match vout.script_pubkey.address() { + Some(TransparentAddress::PublicKey(hash)) => { + if hash[..] == ripemd160::Ripemd160::digest(&Sha256::digest(&pubkey))[..] { + // This is out address. Add this as an output to the txid + self.add_toutput_to_wtx(height, &tx.txid(), &vout, n as u64); + } + }, + _ => {} + } + } + + if total_transparent_spend > 0 { // Update the WalletTx. Do it in a short scope because of the write lock. - self.txs.write().unwrap() - .get_mut(&tx.txid()).unwrap() + let mut txs = self.txs.write().unwrap(); + + if !txs.contains_key(&tx.txid()) { + let tx_entry = WalletTx::new(height, &tx.txid()); + txs.insert(tx.txid().clone(), tx_entry); + } + + txs.get_mut(&tx.txid()).unwrap() .total_transparent_value_spent = total_transparent_spend; } @@ -838,19 +984,14 @@ impl LightWallet { } } - pub fn scan_utxo(&self, utxo: &Utxo) { - // Grab a write lock to the wallet so we can update it. - let mut txs = self.txs.write().unwrap(); + pub fn clear_utxos(&self) { + let mut utxos = self.utxos.write().unwrap(); + utxos.clear(); + } - // If the Txid doesn't exist, create it - if !txs.contains_key(&utxo.txid) { - let tx_entry = WalletTx::new(utxo.height, &utxo.txid); - txs.insert(utxo.txid, tx_entry); - } - let tx_entry = txs.get_mut(&utxo.txid).unwrap(); - - // Add the utxo into the Wallet Tx entry - tx_entry.utxos.push(utxo.clone()); + pub fn add_utxo(&self, utxo: &Utxo) { + let mut utxos = self.utxos.write().unwrap(); + utxos.push(utxo.clone()); } pub fn scan_block(&self, block: &[u8]) -> bool { diff --git a/rust-lightclient/src/main.rs b/rust-lightclient/src/main.rs index 6247727..c1ba335 100644 --- a/rust-lightclient/src/main.rs +++ b/rust-lightclient/src/main.rs @@ -58,6 +58,10 @@ pub fn main() { } }; + if cmd_args.is_empty() { + continue; + } + let cmd = cmd_args.remove(0); let args: Vec<&str> = cmd_args.iter().map(|s| s.as_ref()).collect(); commands::do_user_command(&cmd, &args, &mut lightclient);