Add datetime to transactions

This commit is contained in:
Aditya Kulkarni 2019-10-18 16:54:26 -07:00
parent 5905ef728b
commit ebf3c71339
3 changed files with 103 additions and 86 deletions

View File

@ -7,10 +7,13 @@ use std::sync::{Arc, RwLock};
use std::sync::atomic::{AtomicU64, AtomicI32, AtomicUsize, Ordering};
use std::path::Path;
use std::fs::File;
use std::collections::HashMap;
use std::io;
use std::io::prelude::*;
use std::io::{BufReader, BufWriter, Error, ErrorKind};
use protobuf::parse_from_bytes;
use json::{object, array, JsonValue};
use zcash_primitives::transaction::{TxId, Transaction};
use zcash_client_backend::{
@ -393,6 +396,7 @@ impl LightClient {
} else {
Some(object!{
"created_in_block" => wtx.block,
"datetime" => wtx.datetime,
"created_in_txid" => format!("{}", txid),
"value" => nd.note.value,
"is_change" => nd.is_change,
@ -413,71 +417,50 @@ impl LightClient {
}
});
// Collect UTXOs
let utxos = self.wallet.get_utxos().iter()
.filter(|utxo| utxo.unconfirmed_spent.is_none()) // Filter out unconfirmed from the list of utxos
.map(|utxo| {
object!{
"created_in_block" => utxo.height,
"created_in_txid" => format!("{}", utxo.txid),
"value" => utxo.value,
"scriptkey" => hex::encode(utxo.script.clone()),
"is_change" => false, // TODO: Identify notes as change if we send change to taddrs
"address" => utxo.address.clone(),
"spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)),
"unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)),
}
})
.collect::<Vec<JsonValue>>();
let mut unspent_utxos: Vec<JsonValue> = vec![];
let mut spent_utxos : Vec<JsonValue> = vec![];
let mut pending_utxos: Vec<JsonValue> = vec![];
// Collect pending UTXOs
let pending_utxos = self.wallet.get_utxos().iter()
.filter(|utxo| utxo.unconfirmed_spent.is_some()) // Filter to include only unconfirmed utxos
.map(|utxo|
object!{
"created_in_block" => utxo.height,
"created_in_txid" => format!("{}", utxo.txid),
self.wallet.txs.read().unwrap().iter()
.flat_map( |(txid, wtx)| {
wtx.utxos.iter().filter_map(move |utxo|
if !all_notes && utxo.spent.is_some() {
None
} else {
Some(object!{
"created_in_block" => wtx.block,
"datetime" => wtx.datetime,
"created_in_txid" => format!("{}", txid),
"value" => utxo.value,
"scriptkey" => hex::encode(utxo.script.clone()),
"is_change" => false, // TODO: Identify notes as change if we send change to taddrs
"address" => utxo.address.clone(),
"spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)),
"unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)),
})
}
)
.collect::<Vec<JsonValue>>();
})
.for_each( |utxo| {
if utxo["spent"].is_null() && utxo["unconfirmed_spent"].is_null() {
unspent_utxos.push(utxo);
} else if !utxo["spent"].is_null() {
spent_utxos.push(utxo);
} else {
pending_utxos.push(utxo);
}
});
let mut res = object!{
"unspent_notes" => unspent_notes,
"pending_notes" => pending_notes,
"utxos" => utxos,
"utxos" => unspent_utxos,
"pending_utxos" => pending_utxos,
};
if all_notes {
res["spent_notes"] = JsonValue::Array(spent_notes);
}
// If all notes, also add historical utxos
if all_notes {
res["spent_utxos"] = JsonValue::Array(self.wallet.txs.read().unwrap().values()
.flat_map(|wtx| {
wtx.utxos.iter()
.filter(|utxo| utxo.spent.is_some())
.map(|utxo| {
object!{
"created_in_block" => wtx.block,
"created_in_txid" => format!("{}", utxo.txid),
"value" => utxo.value,
"scriptkey" => hex::encode(utxo.script.clone()),
"is_change" => false, // TODO: Identify notes as change if we send change to taddrs
"address" => utxo.address.clone(),
"spent" => utxo.spent.map(|spent_txid| format!("{}", spent_txid)),
"unconfirmed_spent" => utxo.unconfirmed_spent.map(|spent_txid| format!("{}", spent_txid)),
}
}).collect::<Vec<JsonValue>>()
}).collect::<Vec<JsonValue>>()
);
res["spent_utxos"] = JsonValue::Array(spent_utxos);
}
res
@ -511,6 +494,7 @@ impl LightClient {
txns.push(object! {
"block_height" => v.block,
"datetime" => v.datetime,
"txid" => format!("{}", v.txid),
"amount" => total_change as i64
- v.total_shielded_value_spent as i64
@ -525,6 +509,7 @@ impl LightClient {
.map ( |nd|
object! {
"block_height" => v.block,
"datetime" => v.datetime,
"txid" => format!("{}", v.txid),
"amount" => nd.note.value as i64,
"address" => self.wallet.note_address(nd),
@ -538,6 +523,7 @@ impl LightClient {
// Create an input transaction for the transparent value as well.
txns.push(object!{
"block_height" => v.block,
"datetime" => v.datetime,
"txid" => format!("{}", v.txid),
"amount" => total_transparent_received as i64 - v.total_transparent_value_spent as i64,
"address" => v.utxos.iter().map(|u| u.address.clone()).collect::<Vec<String>>().join(","),
@ -636,6 +622,10 @@ impl LightClient {
// Fetch CompactBlocks in increments
loop {
// Collect all block times, because we'll need to update transparent tx
// datetime via the block height timestamp
let block_times = Arc::new(RwLock::new(HashMap::new()));
let local_light_wallet = self.wallet.clone();
let local_bytes_downloaded = bytes_downloaded.clone();
@ -650,7 +640,9 @@ impl LightClient {
// Fetch compact blocks
info!("Fetching blocks {}-{}", start_height, end_height);
let all_txs = all_new_txs.clone();
let block_times_inner = block_times.clone();
let last_invalid_height = Arc::new(AtomicI32::new(0));
let last_invalid_height_inner = last_invalid_height.clone();
@ -661,8 +653,18 @@ impl LightClient {
return;
}
let block: Result<zcash_client_backend::proto::compact_formats::CompactBlock, _>
= parse_from_bytes(encoded_block);
match block {
Ok(b) => {
block_times_inner.write().unwrap().insert(b.height, b.time);
},
Err(_) => {}
}
match local_light_wallet.scan_block(encoded_block) {
Ok(block_txns) => {
// Add to global tx list
all_txs.write().unwrap().extend_from_slice(&block_txns.iter().map(|txid| (txid.clone(), height as i32)).collect::<Vec<_>>()[..]);
},
Err(invalid_height) => {
@ -711,7 +713,8 @@ impl LightClient {
let tx = Transaction::read(tx_bytes).unwrap();
// Scan this Tx for transparent inputs and outputs
wallet.scan_full_tx(&tx, height as i32);
let datetime = block_times.read().unwrap().get(&height).map(|v| *v).unwrap_or(0);
wallet.scan_full_tx(&tx, height as i32, datetime as u64);
}
);
@ -724,6 +727,7 @@ impl LightClient {
end_height = latest_block;
}
}
if print_updates{
println!(""); // New line to finish up the updates
}
@ -752,7 +756,6 @@ impl LightClient {
// And go and fetch the txids, getting the full transaction, so we can
// read the memos
for (txid, height) in txids_to_fetch {
let light_wallet_clone = self.wallet.clone();
info!("Fetching full Tx: {}", txid);
@ -760,7 +763,7 @@ impl LightClient {
fetch_full_tx(&self.get_server_uri(), txid, self.config.no_cert_verification, move |tx_bytes: &[u8] | {
let tx = Transaction::read(tx_bytes).unwrap();
light_wallet_clone.scan_full_tx(&tx, height);
light_wallet_clone.scan_full_tx(&tx, height, 0);
});
};

View File

@ -560,12 +560,12 @@ impl LightWallet {
.sum::<u64>()
}
fn add_toutput_to_wtx(&self, height: i32, txid: &TxId, vout: &TxOut, n: u64) {
fn add_toutput_to_wtx(&self, height: i32, timestamp: u64, txid: &TxId, vout: &TxOut, n: u64) {
let mut txs = self.txs.write().unwrap();
// Find the existing transaction entry, or create a new one.
if !txs.contains_key(&txid) {
let tx_entry = WalletTx::new(height, &txid);
let tx_entry = WalletTx::new(height, timestamp, &txid);
txs.insert(txid.clone(), tx_entry);
}
let tx_entry = txs.get_mut(&txid).unwrap();
@ -600,7 +600,7 @@ impl LightWallet {
}
// Scan the full Tx and update memos for incoming shielded transactions
pub fn scan_full_tx(&self, tx: &Transaction, height: i32) {
pub fn scan_full_tx(&self, tx: &Transaction, height: i32, datetime: u64) {
// Scan all the inputs to see if we spent any transparent funds in this tx
// TODO: Save this object
@ -639,7 +639,7 @@ impl LightWallet {
let mut txs = self.txs.write().unwrap();
if !txs.contains_key(&tx.txid()) {
let tx_entry = WalletTx::new(height, &tx.txid());
let tx_entry = WalletTx::new(height, datetime, &tx.txid());
txs.insert(tx.txid().clone(), tx_entry);
}
@ -661,7 +661,7 @@ impl LightWallet {
Some(TransparentAddress::PublicKey(hash)) => {
if hash[..] == ripemd160::Ripemd160::digest(&Sha256::digest(&pubkey))[..] {
// This is our address. Add this as an output to the txid
self.add_toutput_to_wtx(height, &tx.txid(), &vout, n as u64);
self.add_toutput_to_wtx(height, datetime, &tx.txid(), &vout, n as u64);
}
},
_ => {}
@ -1013,7 +1013,7 @@ impl LightWallet {
// Find the existing transaction entry, or create a new one.
if !txs.contains_key(&tx.txid) {
let tx_entry = WalletTx::new(block_data.height as i32, &tx.txid);
let tx_entry = WalletTx::new(block_data.height as i32, block.time as u64, &tx.txid);
txs.insert(tx.txid, tx_entry);
}
let tx_entry = txs.get_mut(&tx.txid).unwrap();
@ -1669,7 +1669,7 @@ pub mod tests {
tx.add_t_output(&pk, AMOUNT1);
let txid1 = tx.get_tx().txid();
wallet.scan_full_tx(&tx.get_tx(), 100); // Pretend it is at height 100
wallet.scan_full_tx(&tx.get_tx(), 100, 0); // Pretend it is at height 100
{
let txs = wallet.txs.read().unwrap();
@ -1694,7 +1694,7 @@ pub mod tests {
tx.add_t_input(txid1, 0);
let txid2 = tx.get_tx().txid();
wallet.scan_full_tx(&tx.get_tx(), 101); // Pretent it is at height 101
wallet.scan_full_tx(&tx.get_tx(), 101, 0); // Pretent it is at height 101
{
// Make sure the txid was spent
@ -1741,7 +1741,7 @@ pub mod tests {
tx.add_t_output(&non_wallet_pk, 25);
let txid1 = tx.get_tx().txid();
wallet.scan_full_tx(&tx.get_tx(), 100); // Pretend it is at height 100
wallet.scan_full_tx(&tx.get_tx(), 100, 0); // Pretend it is at height 100
{
let txs = wallet.txs.read().unwrap();
@ -1766,7 +1766,7 @@ pub mod tests {
tx.add_t_input(txid1, 1); // Ours was at position 1 in the input tx
let txid2 = tx.get_tx().txid();
wallet.scan_full_tx(&tx.get_tx(), 101); // Pretent it is at height 101
wallet.scan_full_tx(&tx.get_tx(), 101, 0); // Pretent it is at height 101
{
// Make sure the txid was spent
@ -1815,7 +1815,7 @@ pub mod tests {
let mut tx = FakeTransaction::new_with_txid(txid1);
tx.add_t_output(&pk, TAMOUNT1);
wallet.scan_full_tx(&tx.get_tx(), 0); // Height 0
wallet.scan_full_tx(&tx.get_tx(), 0, 0); // Height 0
const AMOUNT2:u64 = 2;
@ -1828,7 +1828,7 @@ pub mod tests {
let mut tx = FakeTransaction::new_with_txid(txid2);
tx.add_t_input(txid1, 0);
wallet.scan_full_tx(&tx.get_tx(), 1); // Height 1
wallet.scan_full_tx(&tx.get_tx(), 1, 0); // Height 1
// Now, the original note should be spent and there should be a change
assert_eq!(wallet.zbalance(None), AMOUNT1 - AMOUNT2 ); // The t addr amount is received + spent, so it cancels out
@ -2023,7 +2023,7 @@ pub mod tests {
}
// Now, full scan the Tx, which should populate the Outgoing Meta data
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Check Outgoing Metadata
{
@ -2063,7 +2063,7 @@ pub mod tests {
let mut cb3 = FakeCompactBlock::new(2, block_hash);
cb3.add_tx(&sent_tx);
wallet.scan_block(&cb3.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Because the builder will randomize notes outputted, we need to find
// which note number is the change and which is the output note (Because this tx
@ -2116,7 +2116,7 @@ pub mod tests {
let mut cb4 = FakeCompactBlock::new(3, cb3.hash());
cb4.add_tx(&sent_tx);
wallet.scan_block(&cb4.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 3);
wallet.scan_full_tx(&sent_tx, 3, 0);
{
// Both notes should be spent now.
@ -2187,7 +2187,7 @@ pub mod tests {
}
// Now, full scan the Tx, which should populate the Outgoing Meta data
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Check Outgoing Metadata for t address
{
@ -2215,7 +2215,7 @@ pub mod tests {
tx.add_t_output(&pk, AMOUNT_T);
let txid_t = tx.get_tx().txid();
wallet.scan_full_tx(&tx.get_tx(), 1); // Pretend it is at height 1
wallet.scan_full_tx(&tx.get_tx(), 1, 0); // Pretend it is at height 1
{
let txs = wallet.txs.read().unwrap();
@ -2277,7 +2277,7 @@ pub mod tests {
// Scan the compact block and the full Tx
wallet.scan_block(&cb3.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Now this new Spent tx should be in, so the note should be marked confirmed spent
{
@ -2327,7 +2327,7 @@ pub mod tests {
wallet.scan_block(&cb3.as_bytes()).unwrap();
// And scan the Full Tx to get the memo
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
{
let txs = wallet.txs.read().unwrap();
@ -2366,7 +2366,7 @@ pub mod tests {
wallet.scan_block(&cb3.as_bytes()).unwrap();
// And scan the Full Tx to get the memo
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
{
let txs = wallet.txs.read().unwrap();
@ -2424,7 +2424,7 @@ pub mod tests {
let mut cb3 = FakeCompactBlock::new(2, block_hash);
cb3.add_tx(&sent_tx);
wallet.scan_block(&cb3.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Check that the send to the second taddr worked
{
@ -2468,7 +2468,7 @@ pub mod tests {
let mut cb4 = FakeCompactBlock::new(3, cb3.hash());
cb4.add_tx(&sent_tx);
wallet.scan_block(&cb4.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 3);
wallet.scan_full_tx(&sent_tx, 3, 0);
// Quickly check we have it
{
@ -2505,7 +2505,7 @@ pub mod tests {
let mut cb5 = FakeCompactBlock::new(4, cb4.hash());
cb5.add_tx(&sent_tx);
wallet.scan_block(&cb5.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 4);
wallet.scan_full_tx(&sent_tx, 4, 0);
{
let txs = wallet.txs.read().unwrap();
@ -2561,7 +2561,7 @@ pub mod tests {
let mut cb3 = FakeCompactBlock::new(2, block_hash);
cb3.add_tx(&sent_tx);
wallet.scan_block(&cb3.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 2);
wallet.scan_full_tx(&sent_tx, 2, 0);
// Make sure all the outputs are there!
{
@ -2633,7 +2633,7 @@ pub mod tests {
let mut cb4 = FakeCompactBlock::new(3, cb3.hash());
cb4.add_tx(&sent_tx);
wallet.scan_block(&cb4.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 3);
wallet.scan_full_tx(&sent_tx, 3, 0);
// Make sure all the outputs are there!
{
@ -2811,7 +2811,7 @@ pub mod tests {
let mut cb3 = FakeCompactBlock::new(7, blk6_hash);
cb3.add_tx(&sent_tx);
wallet.scan_block(&cb3.as_bytes()).unwrap();
wallet.scan_full_tx(&sent_tx, 7);
wallet.scan_full_tx(&sent_tx, 7, 0);
// Make sure the Tx is in.
{

View File

@ -362,8 +362,12 @@ impl OutgoingTxMetadata {
}
pub struct WalletTx {
// Block in which this tx was included
pub block: i32,
// Timestamp of Tx. Added in v4
pub datetime: u64,
// Txid of this transaction. It's duplicated here (It is also the Key in the HashMap that points to this
// WalletTx in LightWallet::txs)
pub txid: TxId,
@ -392,12 +396,13 @@ pub struct WalletTx {
impl WalletTx {
pub fn serialized_version() -> u64 {
return 3;
return 4;
}
pub fn new(height: i32, txid: &TxId) -> Self {
pub fn new(height: i32, datetime: u64, txid: &TxId) -> Self {
WalletTx {
block: height,
datetime,
txid: txid.clone(),
notes: vec![],
utxos: vec![],
@ -414,6 +419,12 @@ impl WalletTx {
let block = reader.read_i32::<LittleEndian>()?;
let datetime = if version >= 4 {
reader.read_u64::<LittleEndian>()?
} else {
0
};
let mut txid_bytes = [0u8; 32];
reader.read_exact(&mut txid_bytes)?;
@ -432,6 +443,7 @@ impl WalletTx {
Ok(WalletTx{
block,
datetime,
txid,
notes,
utxos,
@ -447,6 +459,8 @@ impl WalletTx {
writer.write_i32::<LittleEndian>(self.block)?;
writer.write_u64::<LittleEndian>(self.datetime)?;
writer.write_all(&self.txid.0)?;
Vector::write(&mut writer, &self.notes, |w, nd| nd.write(w))?;