diff --git a/lib/src/lightclient.rs b/lib/src/lightclient.rs index e527c2d..b19f2f6 100644 --- a/lib/src/lightclient.rs +++ b/lib/src/lightclient.rs @@ -7,10 +7,13 @@ use std::sync::{Arc, RwLock}; use std::sync::atomic::{AtomicU64, AtomicI32, AtomicUsize, Ordering}; use std::path::Path; use std::fs::File; +use std::collections::HashMap; use std::io; use std::io::prelude::*; use std::io::{BufReader, BufWriter, Error, ErrorKind}; +use protobuf::parse_from_bytes; + use json::{object, array, JsonValue}; use zcash_primitives::transaction::{TxId, Transaction}; use zcash_client_backend::{ @@ -393,6 +396,7 @@ impl LightClient { } else { Some(object!{ "created_in_block" => wtx.block, + "datetime" => wtx.datetime, "created_in_txid" => format!("{}", txid), "value" => nd.note.value, "is_change" => nd.is_change, @@ -413,71 +417,50 @@ impl LightClient { } }); - // Collect UTXOs - let utxos = self.wallet.get_utxos().iter() - .filter(|utxo| utxo.unconfirmed_spent.is_none()) // Filter out unconfirmed from the list of utxos - .map(|utxo| { - object!{ - "created_in_block" => utxo.height, - "created_in_txid" => format!("{}", utxo.txid), - "value" => utxo.value, - "scriptkey" => hex::encode(utxo.script.clone()), - "is_change" => false, // TODO: Identify notes as change if we send change to taddrs - "address" => utxo.address.clone(), - "spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)), - "unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)), - } - }) - .collect::>(); + let mut unspent_utxos: Vec = vec![]; + let mut spent_utxos : Vec = vec![]; + let mut pending_utxos: Vec = vec![]; - // Collect pending UTXOs - let pending_utxos = self.wallet.get_utxos().iter() - .filter(|utxo| utxo.unconfirmed_spent.is_some()) // Filter to include only unconfirmed utxos - .map(|utxo| - object!{ - "created_in_block" => utxo.height, - "created_in_txid" => format!("{}", utxo.txid), - "value" => utxo.value, - "scriptkey" => hex::encode(utxo.script.clone()), - "is_change" => false, // TODO: Identify notes as change if we send change to taddrs - "address" => utxo.address.clone(), - "spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)), - "unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)), + self.wallet.txs.read().unwrap().iter() + .flat_map( |(txid, wtx)| { + wtx.utxos.iter().filter_map(move |utxo| + if !all_notes && utxo.spent.is_some() { + None + } else { + Some(object!{ + "created_in_block" => wtx.block, + "datetime" => wtx.datetime, + "created_in_txid" => format!("{}", txid), + "value" => utxo.value, + "scriptkey" => hex::encode(utxo.script.clone()), + "is_change" => false, // TODO: Identify notes as change if we send change to taddrs + "address" => utxo.address.clone(), + "spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)), + "unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)), + }) + } + ) + }) + .for_each( |utxo| { + if utxo["spent"].is_null() && utxo["unconfirmed_spent"].is_null() { + unspent_utxos.push(utxo); + } else if !utxo["spent"].is_null() { + spent_utxos.push(utxo); + } else { + pending_utxos.push(utxo); } - ) - .collect::>(); + }); let mut res = object!{ "unspent_notes" => unspent_notes, "pending_notes" => pending_notes, - "utxos" => utxos, + "utxos" => unspent_utxos, "pending_utxos" => pending_utxos, }; if all_notes { res["spent_notes"] = JsonValue::Array(spent_notes); - } - - // If all notes, also add historical utxos - if all_notes { - res["spent_utxos"] = JsonValue::Array(self.wallet.txs.read().unwrap().values() - .flat_map(|wtx| { - wtx.utxos.iter() - .filter(|utxo| utxo.spent.is_some()) - .map(|utxo| { - object!{ - "created_in_block" => wtx.block, - "created_in_txid" => format!("{}", utxo.txid), - "value" => utxo.value, - "scriptkey" => hex::encode(utxo.script.clone()), - "is_change" => false, // TODO: Identify notes as change if we send change to taddrs - "address" => utxo.address.clone(), - "spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)), - "unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)), - } - }).collect::>() - }).collect::>() - ); + res["spent_utxos"] = JsonValue::Array(spent_utxos); } res @@ -511,6 +494,7 @@ impl LightClient { txns.push(object! { "block_height" => v.block, + "datetime" => v.datetime, "txid" => format!("{}", v.txid), "amount" => total_change as i64 - v.total_shielded_value_spent as i64 @@ -525,6 +509,7 @@ impl LightClient { .map ( |nd| object! { "block_height" => v.block, + "datetime" => v.datetime, "txid" => format!("{}", v.txid), "amount" => nd.note.value as i64, "address" => self.wallet.note_address(nd), @@ -538,6 +523,7 @@ impl LightClient { // Create an input transaction for the transparent value as well. txns.push(object!{ "block_height" => v.block, + "datetime" => v.datetime, "txid" => format!("{}", v.txid), "amount" => total_transparent_received as i64 - v.total_transparent_value_spent as i64, "address" => v.utxos.iter().map(|u| u.address.clone()).collect::>().join(","), @@ -636,6 +622,10 @@ impl LightClient { // Fetch CompactBlocks in increments loop { + // Collect all block times, because we'll need to update transparent tx + // datetime via the block height timestamp + let block_times = Arc::new(RwLock::new(HashMap::new())); + let local_light_wallet = self.wallet.clone(); let local_bytes_downloaded = bytes_downloaded.clone(); @@ -650,7 +640,9 @@ impl LightClient { // Fetch compact blocks info!("Fetching blocks {}-{}", start_height, end_height); + let all_txs = all_new_txs.clone(); + let block_times_inner = block_times.clone(); let last_invalid_height = Arc::new(AtomicI32::new(0)); let last_invalid_height_inner = last_invalid_height.clone(); @@ -661,8 +653,18 @@ impl LightClient { return; } + let block: Result + = parse_from_bytes(encoded_block); + match block { + Ok(b) => { + block_times_inner.write().unwrap().insert(b.height, b.time); + }, + Err(_) => {} + } + match local_light_wallet.scan_block(encoded_block) { Ok(block_txns) => { + // Add to global tx list all_txs.write().unwrap().extend_from_slice(&block_txns.iter().map(|txid| (txid.clone(), height as i32)).collect::>()[..]); }, Err(invalid_height) => { @@ -711,7 +713,8 @@ impl LightClient { let tx = Transaction::read(tx_bytes).unwrap(); // Scan this Tx for transparent inputs and outputs - wallet.scan_full_tx(&tx, height as i32); + let datetime = block_times.read().unwrap().get(&height).map(|v| *v).unwrap_or(0); + wallet.scan_full_tx(&tx, height as i32, datetime as u64); } ); @@ -722,8 +725,9 @@ impl LightClient { break; } else if end_height > latest_block { end_height = latest_block; - } + } } + if print_updates{ println!(""); // New line to finish up the updates } @@ -752,7 +756,6 @@ impl LightClient { // And go and fetch the txids, getting the full transaction, so we can // read the memos - for (txid, height) in txids_to_fetch { let light_wallet_clone = self.wallet.clone(); info!("Fetching full Tx: {}", txid); @@ -760,7 +763,7 @@ impl LightClient { fetch_full_tx(&self.get_server_uri(), txid, self.config.no_cert_verification, move |tx_bytes: &[u8] | { let tx = Transaction::read(tx_bytes).unwrap(); - light_wallet_clone.scan_full_tx(&tx, height); + light_wallet_clone.scan_full_tx(&tx, height, 0); }); }; diff --git a/lib/src/lightwallet.rs b/lib/src/lightwallet.rs index 80b0d86..0db9278 100644 --- a/lib/src/lightwallet.rs +++ b/lib/src/lightwallet.rs @@ -560,12 +560,12 @@ impl LightWallet { .sum::() } - fn add_toutput_to_wtx(&self, height: i32, txid: &TxId, vout: &TxOut, n: u64) { + fn add_toutput_to_wtx(&self, height: i32, timestamp: u64, txid: &TxId, vout: &TxOut, n: u64) { let mut txs = self.txs.write().unwrap(); // Find the existing transaction entry, or create a new one. if !txs.contains_key(&txid) { - let tx_entry = WalletTx::new(height, &txid); + let tx_entry = WalletTx::new(height, timestamp, &txid); txs.insert(txid.clone(), tx_entry); } let tx_entry = txs.get_mut(&txid).unwrap(); @@ -600,7 +600,7 @@ impl LightWallet { } // Scan the full Tx and update memos for incoming shielded transactions - pub fn scan_full_tx(&self, tx: &Transaction, height: i32) { + pub fn scan_full_tx(&self, tx: &Transaction, height: i32, datetime: u64) { // Scan all the inputs to see if we spent any transparent funds in this tx // TODO: Save this object @@ -639,7 +639,7 @@ impl LightWallet { let mut txs = self.txs.write().unwrap(); if !txs.contains_key(&tx.txid()) { - let tx_entry = WalletTx::new(height, &tx.txid()); + let tx_entry = WalletTx::new(height, datetime, &tx.txid()); txs.insert(tx.txid().clone(), tx_entry); } @@ -661,7 +661,7 @@ impl LightWallet { Some(TransparentAddress::PublicKey(hash)) => { if hash[..] == ripemd160::Ripemd160::digest(&Sha256::digest(&pubkey))[..] { // This is our address. Add this as an output to the txid - self.add_toutput_to_wtx(height, &tx.txid(), &vout, n as u64); + self.add_toutput_to_wtx(height, datetime, &tx.txid(), &vout, n as u64); } }, _ => {} @@ -1013,7 +1013,7 @@ impl LightWallet { // Find the existing transaction entry, or create a new one. if !txs.contains_key(&tx.txid) { - let tx_entry = WalletTx::new(block_data.height as i32, &tx.txid); + let tx_entry = WalletTx::new(block_data.height as i32, block.time as u64, &tx.txid); txs.insert(tx.txid, tx_entry); } let tx_entry = txs.get_mut(&tx.txid).unwrap(); @@ -1669,7 +1669,7 @@ pub mod tests { tx.add_t_output(&pk, AMOUNT1); let txid1 = tx.get_tx().txid(); - wallet.scan_full_tx(&tx.get_tx(), 100); // Pretend it is at height 100 + wallet.scan_full_tx(&tx.get_tx(), 100, 0); // Pretend it is at height 100 { let txs = wallet.txs.read().unwrap(); @@ -1694,7 +1694,7 @@ pub mod tests { tx.add_t_input(txid1, 0); let txid2 = tx.get_tx().txid(); - wallet.scan_full_tx(&tx.get_tx(), 101); // Pretent it is at height 101 + wallet.scan_full_tx(&tx.get_tx(), 101, 0); // Pretent it is at height 101 { // Make sure the txid was spent @@ -1741,7 +1741,7 @@ pub mod tests { tx.add_t_output(&non_wallet_pk, 25); let txid1 = tx.get_tx().txid(); - wallet.scan_full_tx(&tx.get_tx(), 100); // Pretend it is at height 100 + wallet.scan_full_tx(&tx.get_tx(), 100, 0); // Pretend it is at height 100 { let txs = wallet.txs.read().unwrap(); @@ -1766,7 +1766,7 @@ pub mod tests { tx.add_t_input(txid1, 1); // Ours was at position 1 in the input tx let txid2 = tx.get_tx().txid(); - wallet.scan_full_tx(&tx.get_tx(), 101); // Pretent it is at height 101 + wallet.scan_full_tx(&tx.get_tx(), 101, 0); // Pretent it is at height 101 { // Make sure the txid was spent @@ -1815,7 +1815,7 @@ pub mod tests { let mut tx = FakeTransaction::new_with_txid(txid1); tx.add_t_output(&pk, TAMOUNT1); - wallet.scan_full_tx(&tx.get_tx(), 0); // Height 0 + wallet.scan_full_tx(&tx.get_tx(), 0, 0); // Height 0 const AMOUNT2:u64 = 2; @@ -1828,7 +1828,7 @@ pub mod tests { let mut tx = FakeTransaction::new_with_txid(txid2); tx.add_t_input(txid1, 0); - wallet.scan_full_tx(&tx.get_tx(), 1); // Height 1 + wallet.scan_full_tx(&tx.get_tx(), 1, 0); // Height 1 // Now, the original note should be spent and there should be a change assert_eq!(wallet.zbalance(None), AMOUNT1 - AMOUNT2 ); // The t addr amount is received + spent, so it cancels out @@ -2023,7 +2023,7 @@ pub mod tests { } // Now, full scan the Tx, which should populate the Outgoing Meta data - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Check Outgoing Metadata { @@ -2063,7 +2063,7 @@ pub mod tests { let mut cb3 = FakeCompactBlock::new(2, block_hash); cb3.add_tx(&sent_tx); wallet.scan_block(&cb3.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Because the builder will randomize notes outputted, we need to find // which note number is the change and which is the output note (Because this tx @@ -2116,7 +2116,7 @@ pub mod tests { let mut cb4 = FakeCompactBlock::new(3, cb3.hash()); cb4.add_tx(&sent_tx); wallet.scan_block(&cb4.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 3); + wallet.scan_full_tx(&sent_tx, 3, 0); { // Both notes should be spent now. @@ -2187,7 +2187,7 @@ pub mod tests { } // Now, full scan the Tx, which should populate the Outgoing Meta data - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Check Outgoing Metadata for t address { @@ -2215,7 +2215,7 @@ pub mod tests { tx.add_t_output(&pk, AMOUNT_T); let txid_t = tx.get_tx().txid(); - wallet.scan_full_tx(&tx.get_tx(), 1); // Pretend it is at height 1 + wallet.scan_full_tx(&tx.get_tx(), 1, 0); // Pretend it is at height 1 { let txs = wallet.txs.read().unwrap(); @@ -2277,7 +2277,7 @@ pub mod tests { // Scan the compact block and the full Tx wallet.scan_block(&cb3.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Now this new Spent tx should be in, so the note should be marked confirmed spent { @@ -2327,7 +2327,7 @@ pub mod tests { wallet.scan_block(&cb3.as_bytes()).unwrap(); // And scan the Full Tx to get the memo - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); { let txs = wallet.txs.read().unwrap(); @@ -2366,7 +2366,7 @@ pub mod tests { wallet.scan_block(&cb3.as_bytes()).unwrap(); // And scan the Full Tx to get the memo - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); { let txs = wallet.txs.read().unwrap(); @@ -2424,7 +2424,7 @@ pub mod tests { let mut cb3 = FakeCompactBlock::new(2, block_hash); cb3.add_tx(&sent_tx); wallet.scan_block(&cb3.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Check that the send to the second taddr worked { @@ -2468,7 +2468,7 @@ pub mod tests { let mut cb4 = FakeCompactBlock::new(3, cb3.hash()); cb4.add_tx(&sent_tx); wallet.scan_block(&cb4.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 3); + wallet.scan_full_tx(&sent_tx, 3, 0); // Quickly check we have it { @@ -2505,7 +2505,7 @@ pub mod tests { let mut cb5 = FakeCompactBlock::new(4, cb4.hash()); cb5.add_tx(&sent_tx); wallet.scan_block(&cb5.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 4); + wallet.scan_full_tx(&sent_tx, 4, 0); { let txs = wallet.txs.read().unwrap(); @@ -2561,7 +2561,7 @@ pub mod tests { let mut cb3 = FakeCompactBlock::new(2, block_hash); cb3.add_tx(&sent_tx); wallet.scan_block(&cb3.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 2); + wallet.scan_full_tx(&sent_tx, 2, 0); // Make sure all the outputs are there! { @@ -2633,7 +2633,7 @@ pub mod tests { let mut cb4 = FakeCompactBlock::new(3, cb3.hash()); cb4.add_tx(&sent_tx); wallet.scan_block(&cb4.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 3); + wallet.scan_full_tx(&sent_tx, 3, 0); // Make sure all the outputs are there! { @@ -2811,7 +2811,7 @@ pub mod tests { let mut cb3 = FakeCompactBlock::new(7, blk6_hash); cb3.add_tx(&sent_tx); wallet.scan_block(&cb3.as_bytes()).unwrap(); - wallet.scan_full_tx(&sent_tx, 7); + wallet.scan_full_tx(&sent_tx, 7, 0); // Make sure the Tx is in. { diff --git a/lib/src/lightwallet/data.rs b/lib/src/lightwallet/data.rs index 5a2a586..8660f5d 100644 --- a/lib/src/lightwallet/data.rs +++ b/lib/src/lightwallet/data.rs @@ -362,8 +362,12 @@ impl OutgoingTxMetadata { } pub struct WalletTx { + // Block in which this tx was included pub block: i32, + // Timestamp of Tx. Added in v4 + pub datetime: u64, + // Txid of this transaction. It's duplicated here (It is also the Key in the HashMap that points to this // WalletTx in LightWallet::txs) pub txid: TxId, @@ -392,12 +396,13 @@ pub struct WalletTx { impl WalletTx { pub fn serialized_version() -> u64 { - return 3; + return 4; } - pub fn new(height: i32, txid: &TxId) -> Self { + pub fn new(height: i32, datetime: u64, txid: &TxId) -> Self { WalletTx { block: height, + datetime, txid: txid.clone(), notes: vec![], utxos: vec![], @@ -414,6 +419,12 @@ impl WalletTx { let block = reader.read_i32::()?; + let datetime = if version >= 4 { + reader.read_u64::()? + } else { + 0 + }; + let mut txid_bytes = [0u8; 32]; reader.read_exact(&mut txid_bytes)?; @@ -432,6 +443,7 @@ impl WalletTx { Ok(WalletTx{ block, + datetime, txid, notes, utxos, @@ -447,6 +459,8 @@ impl WalletTx { writer.write_i32::(self.block)?; + writer.write_u64::(self.datetime)?; + writer.write_all(&self.txid.0)?; Vector::write(&mut writer, &self.notes, |w, nd| nd.write(w))?;