Add block cache trait

Cargo.toml

@@ -114,6 +114,9 @@ aes = "0.8"
 fpe = "0.6"
 zip32 = "0.1"
 
+# Runtime
+tokio = { version = "1.21.0", features = ["rt-multi-thread"] }
+
 [profile.release]
 lto = true
 panic = 'abort'

zcash_client_backend/CHANGELOG.md

@@ -20,6 +20,7 @@ and this library adheres to Rust's notion of
   - `ORCHARD_SHARD_HEIGHT`
   - `BlockMetadata::orchard_tree_size`
   - `chain::ScanSummary::{spent_orchard_note_count, received_orchard_note_count}`
+  - `chain::BlockCache` trait
 - `zcash_client_backend::fees`:
   - `orchard`
   - `ChangeValue::orchard`
@@ -33,6 +34,8 @@ and this library adheres to Rust's notion of
   - `Nullifiers::{orchard, extend_orchard, retain_orchard}`
   - `TaggedOrchardBatch`
   - `TaggedOrchardBatchRunner`
+  - `testing` module
+  - `testing::{'fake_compact_block`, `random_compact_tx`} (moved from `tests` module).
 - `zcash_client_backend::wallet`:
   - `Note::Orchard`
   - `WalletOrchardSpend`
@@ -53,6 +56,8 @@ and this library adheres to Rust's notion of
   - Added method `WalletRead::validate_seed`
 - `zcash_client_backend::fees`:
   - Arguments to `ChangeStrategy::compute_balance` have changed.
+- `zcash_client_backend::scanning`:
+  - `testing::fake_compact_block` is now public.
 - `zcash_client_backend::zip321::render::amount_str` now takes a
   `NonNegativeAmount` rather than a signed `Amount` as its argument.
 - `zcash_client_backend::zip321::parse::parse_amount` now parses a

zcash_client_backend/Cargo.toml

@@ -75,10 +75,14 @@ shardtree.workspace = true
 
 # - Test dependencies
 proptest = { workspace = true, optional = true }
+jubjub = { workspace = true, optional = true }
 
 # - ZIP 321
 nom = "7"
 
+# - Runtime
+tokio.workspace = true
+
 # Dependencies used internally:
 # (Breaking upgrades to these are usually backwards-compatible, but check MSRVs.)
 # - Documentation
@@ -127,6 +131,7 @@ orchard = ["dep:orchard", "zcash_keys/orchard"]
 ## Exposes APIs that are useful for testing, such as `proptest` strategies.
 test-dependencies = [
     "dep:proptest",
+    "dep:jubjub",
     "orchard?/test-dependencies",
     "zcash_keys/test-dependencies",
     "zcash_primitives/test-dependencies",

zcash_client_backend/src/data_api/chain.rs

@@ -146,10 +146,11 @@
 use std::ops::Range;
 
 use subtle::ConditionallySelectable;
+use tokio::task::JoinHandle;
 use zcash_primitives::consensus::{self, BlockHeight};
 
 use crate::{
-    data_api::{NullifierQuery, WalletWrite},
+    data_api::{scanning::ScanRange, NullifierQuery, WalletWrite},
     proto::compact_formats::CompactBlock,
     scanning::{scan_block_with_runners, BatchRunners, Nullifiers, ScanningKeys},
 };
@@ -211,6 +212,178 @@ pub trait BlockSource {
         F: FnMut(CompactBlock) -> Result<(), error::Error<WalletErrT, Self::Error>>;
 }
 
+/// `BlockCache` is a trait that extends `BlockSource` and defines methods for managing
+/// a cache of compact blocks.
+///
+/// # Examples
+///
+/// ```
+///    use std::sync::{Arc, Mutex};
+///    use tokio::task::JoinHandle;
+///    use zcash_client_backend::data_api::{
+///        chain::{error, BlockCache, BlockSource},
+///        scanning::{ScanPriority, ScanRange},
+///    };
+///    use zcash_client_backend::proto::compact_formats::CompactBlock;
+///    use zcash_primitives::consensus::BlockHeight;
+///
+///    struct ExampleBlockCache {
+///        cached_blocks: Arc<Mutex<Vec<CompactBlock>>>,
+///    }
+///
+/// #    impl BlockSource for ExampleBlockCache {
+/// #        type Error = ();
+/// #
+/// #        fn with_blocks<F, WalletErrT>(
+/// #            &self,
+/// #            _from_height: Option<BlockHeight>,
+/// #            _limit: Option<usize>,
+/// #            _with_block: F,
+/// #        ) -> Result<(), error::Error<WalletErrT, Self::Error>>
+/// #        where
+/// #            F: FnMut(CompactBlock) -> Result<(), error::Error<WalletErrT, Self::Error>>,
+/// #        {
+/// #            Ok(())
+/// #        }
+/// #    }
+/// #
+///    impl BlockCache for ExampleBlockCache {
+///        fn read(&self, range: &ScanRange) -> Result<Vec<CompactBlock>, Self::Error> {
+///            Ok(self
+///                .cached_blocks
+///                .lock()
+///                .unwrap()
+///                .iter()
+///                .filter(|block| {
+///                    let block_height = BlockHeight::from_u32(block.height as u32);
+///                    range.block_range().contains(&block_height)
+///                })
+///                .cloned()
+///                .collect())
+///        }
+///
+///        fn cache_tip(&self, range: Option<&ScanRange>) -> Result<Option<BlockHeight>, Self::Error> {
+///            let cached_blocks = self.cached_blocks.lock().unwrap();
+///            let blocks: Vec<&CompactBlock> = match range {
+///                Some(range) => cached_blocks
+///                    .iter()
+///                    .filter(|&block| {
+///                        let block_height = BlockHeight::from_u32(block.height as u32);
+///                        range.block_range().contains(&block_height)
+///                    })
+///                    .collect(),
+///                None => cached_blocks.iter().collect(),
+///            };
+///            let highest_block = blocks.iter().max_by_key(|&&block| block.height);
+///            Ok(highest_block.map(|&block| BlockHeight::from_u32(block.height as u32)))
+///        }
+///
+///        fn insert(&self, mut compact_blocks: Vec<CompactBlock>) -> Result<(), Self::Error> {
+///            self.cached_blocks
+///                .lock()
+///                .unwrap()
+///                .append(&mut compact_blocks);
+///            Ok(())
+///        }
+///
+///        fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error> {
+///            self.cached_blocks
+///                .lock()
+///                .unwrap()
+///                .retain(|block| block.height <= block_height.into());
+///            Ok(())
+///        }
+///
+///        fn delete(&self, range: &ScanRange) -> JoinHandle<Result<(), Self::Error>> {
+///            let cached_blocks = Arc::clone(&self.cached_blocks);
+///            let range = range.block_range().clone();
+///            tokio::spawn(async move {
+///                cached_blocks
+///                    .lock()
+///                    .unwrap()
+///                    .retain(|block| !range.contains(&BlockHeight::from_u32(block.height as u32)));
+///                Ok(())
+///            })
+///        }
+///    }
+///
+///    // Example usage
+///    let mut block_cache = ExampleBlockCache {
+///        cached_blocks: Arc::new(Mutex::new(Vec::new())),
+///    };
+///    let range = ScanRange::from_parts(
+///        BlockHeight::from_u32(1)..BlockHeight::from_u32(3),
+///        ScanPriority::Historic,
+///    );
+/// #    let extsk = sapling::zip32::ExtendedSpendingKey::master(&[]);
+/// #    let dfvk = extsk.to_diversifiable_full_viewing_key();
+/// #    let compact_block1 = zcash_client_backend::scanning::testing::fake_compact_block(
+/// #        1u32.into(),
+/// #        zcash_primitives::block::BlockHash([0; 32]),
+/// #        sapling::Nullifier([0; 32]),
+/// #        &dfvk,
+/// #        zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5),
+/// #        false,
+/// #        None,
+/// #    );
+/// #    let compact_block2 = zcash_client_backend::scanning::testing::fake_compact_block(
+/// #        2u32.into(),
+/// #        zcash_primitives::block::BlockHash([0; 32]),
+/// #        sapling::Nullifier([0; 32]),
+/// #        &dfvk,
+/// #        zcash_primitives::transaction::components::amount::NonNegativeAmount::const_from_u64(5),
+/// #        false,
+/// #        None,
+/// #    );
+///    let compact_blocks = vec![compact_block1, compact_block2];
+///
+///    // Insert blocks into the block cache
+///    block_cache.insert(compact_blocks.clone()).unwrap();
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 2);
+///
+///    // Find highest block in the block cache
+///    let cache_tip = block_cache.cache_tip(None).unwrap();
+///    assert_eq!(cache_tip, Some(BlockHeight::from_u32(2)));
+///
+///    // Read from the block cache
+///    let blocks_from_cache = block_cache.read(&range).unwrap();
+///    assert_eq!(blocks_from_cache, compact_blocks);
+///
+///    // Truncate the block cache
+///    block_cache.truncate(BlockHeight::from_u32(1)).unwrap();
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 1);
+///    assert_eq!(
+///        block_cache.cache_tip(None).unwrap(),
+///        Some(BlockHeight::from_u32(1))
+///    );
+///
+///    // Delete blocks from the block cache
+///    let rt = tokio::runtime::Runtime::new().unwrap();
+///    rt.block_on(async {
+///        block_cache.delete(&range).await.unwrap();
+///    });
+///    assert_eq!(block_cache.cached_blocks.lock().unwrap().len(), 0);
+///    assert_eq!(block_cache.cache_tip(None).unwrap(), None);
+/// ```
+pub trait BlockCache: BlockSource + Send + Sync {
+    /// Returns a range of compact blocks from the cache.
+    fn read(&self, range: &ScanRange) -> Result<Vec<CompactBlock>, Self::Error>;
+
+    /// Returns the height of highest block known to the block cache within a specified range.
+    /// If `range` is `None`, returns the tip of the entire cache.
+    fn cache_tip(&self, range: Option<&ScanRange>) -> Result<Option<BlockHeight>, Self::Error>;
+
+    /// Inserts a set of compact blocks into the block cache.
+    fn insert(&self, compact_blocks: Vec<CompactBlock>) -> Result<(), Self::Error>;
+
+    /// Removes all cached blocks above a specified block height.
+    fn truncate(&self, block_height: BlockHeight) -> Result<(), Self::Error>;
+
+    /// Deletes a range of compact blocks from the block cache.
+    /// Returns a `JoinHandle` from a `tokio::spawn` task.
+    fn delete(&self, range: &ScanRange) -> JoinHandle<Result<(), Self::Error>>;
+}
+
 /// Metadata about modifications to the wallet state made in the course of scanning a set of
 /// blocks.
 #[derive(Clone, Debug)]

zcash_client_backend/src/scanning.rs

@@ -1125,16 +1125,12 @@ fn find_received<
     (shielded_outputs, note_commitments)
 }
 
-#[cfg(test)]
-mod tests {
-
-    use std::convert::Infallible;
-
+#[cfg(any(test, feature = "test-dependencies"))]
+pub mod testing {
     use group::{
         ff::{Field, PrimeField},
         GroupEncoding,
     };
-    use incrementalmerkletree::{Position, Retention};
     use rand_core::{OsRng, RngCore};
     use sapling::{
         constants::SPENDING_KEY_GENERATOR,
@@ -1144,26 +1140,18 @@ mod tests {
         zip32::DiversifiableFullViewingKey,
         Nullifier,
     };
-    use zcash_keys::keys::UnifiedSpendingKey;
     use zcash_note_encryption::{Domain, COMPACT_NOTE_SIZE};
     use zcash_primitives::{
         block::BlockHash,
         consensus::{BlockHeight, Network},
         memo::MemoBytes,
         transaction::components::{amount::NonNegativeAmount, sapling::zip212_enforcement},
-        zip32::AccountId,
     };
 
-    use crate::{
-        data_api::BlockMetadata,
-        proto::compact_formats::{
-            self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
-        },
-        scanning::{BatchRunners, ScanningKeys},
+    use crate::proto::compact_formats::{
+        self as compact, CompactBlock, CompactSaplingOutput, CompactSaplingSpend, CompactTx,
     };
 
-    use super::{scan_block, scan_block_with_runners, Nullifiers};
-
     fn random_compact_tx(mut rng: impl RngCore) -> CompactTx {
         let fake_nf = {
             let mut nf = vec![0; 32];
@@ -1202,7 +1190,7 @@ mod tests {
     ///
     /// Set `initial_tree_sizes` to `None` to simulate a `CompactBlock` retrieved
     /// from a `lightwalletd` that is not currently tracking note commitment tree sizes.
-    fn fake_compact_block(
+    pub fn fake_compact_block(
         height: BlockHeight,
         prev_hash: BlockHash,
         nf: Nullifier,
@@ -1281,6 +1269,29 @@ mod tests {
 
         cb
     }
+}
+
+#[cfg(test)]
+mod tests {
+
+    use std::convert::Infallible;
+
+    use incrementalmerkletree::{Position, Retention};
+    use sapling::Nullifier;
+    use zcash_keys::keys::UnifiedSpendingKey;
+    use zcash_primitives::{
+        block::BlockHash,
+        consensus::{BlockHeight, Network},
+        transaction::components::amount::NonNegativeAmount,
+        zip32::AccountId,
+    };
+
+    use crate::{
+        data_api::BlockMetadata,
+        scanning::{BatchRunners, ScanningKeys},
+    };
+
+    use super::{scan_block, scan_block_with_runners, testing::fake_compact_block, Nullifiers};
 
     #[test]
     fn scan_block_with_my_tx() {