From 19d7a7523ad5d62f157f9aad9f8677b8c8336c71 Mon Sep 17 00:00:00 2001 From: darshan <8559992+lakshya-sky@users.noreply.github.com> Date: Thu, 27 Feb 2025 13:43:07 +0530 Subject: [PATCH 1/3] initial design for sharing txn across multiple insertion --- crates/storage/store_db/libmdbx.rs | 64 ++++++++++++++++++++++++++---- 1 file changed, 56 insertions(+), 8 deletions(-) diff --git a/crates/storage/store_db/libmdbx.rs b/crates/storage/store_db/libmdbx.rs index cd3869a00..a0ad62c52 100644 --- a/crates/storage/store_db/libmdbx.rs +++ b/crates/storage/store_db/libmdbx.rs @@ -70,6 +70,40 @@ impl Store { txn.commit().map_err(StoreError::LibmdbxError) } + fn write_batch_with_txn( + txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + key_values: impl Iterator, + ) -> Result<(), StoreError> { + for (key, value) in key_values { + txn.upsert::(key, value) + .map_err(StoreError::LibmdbxError)?; + } + + Ok(()) + } + + fn begin(&self) -> Result, StoreError> { + let txn = self.db.begin_readwrite().map_err(StoreError::LibmdbxError); + txn + } + + fn commit(&self, txn: libmdbx::orm::Transaction<'_, libmdbx::RW>) -> Result<(), StoreError> { + txn.commit().map_err(StoreError::LibmdbxError) + } + + fn rollback(&self, txn: libmdbx::orm::Transaction<'_, libmdbx::RW>) { + drop(txn); + } + + fn write_with_closure( + &self, + f: impl FnOnce(&mut libmdbx::orm::Transaction<'_, libmdbx::RW>) -> Result<(), StoreError>, + ) -> Result<(), StoreError> { + let mut txn = self.begin()?; + f(&mut txn)?; + self.commit(txn) + } + // Helper method to read from a libmdbx table fn read(&self, key: T::Key) -> Result, StoreError> { let txn = self.db.begin_read().map_err(StoreError::LibmdbxError)?; @@ -82,6 +116,27 @@ impl Store { ) -> Result, StoreError> { Ok(self.read::(number)?.map(|a| a.to())) } + + fn add_transaction_locations_with_txn( + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + write_txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + ) -> Result<(), StoreError> { + #[allow(clippy::type_complexity)] + let key_values = locations + .into_iter() + .map(|(tx_hash, block_number, block_hash, index)| { + (tx_hash.into(), (block_number, block_hash, index).into()) + }); + + Self::write_batch_with_txn::(write_txn, key_values) + } + + fn add_transaction_locations( + &self, + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + ) -> Result<(), StoreError> { + self.write_with_closure(move |txn| Self::add_transaction_locations_with_txn(locations, txn)) + } } impl StoreEngine for Store { @@ -434,14 +489,7 @@ impl StoreEngine for Store { &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError> { - #[allow(clippy::type_complexity)] - let key_values = locations - .into_iter() - .map(|(tx_hash, block_number, block_hash, index)| { - (tx_hash.into(), (block_number, block_hash, index).into()) - }); - - self.write_batch::(key_values) + self.add_transaction_locations(locations) } fn add_receipts( From 9e10d76a92e1c7da81fb12337bb3f2dc32df1a2f Mon Sep 17 00:00:00 2001 From: darshan <8559992+lakshya-sky@users.noreply.github.com> Date: Thu, 27 Feb 2025 14:24:04 +0530 Subject: [PATCH 2/3] add a new method on mdbx to store block body and header --- crates/storage/store_db/libmdbx.rs | 44 ++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/storage/store_db/libmdbx.rs b/crates/storage/store_db/libmdbx.rs index a0ad62c52..98c079a81 100644 --- a/crates/storage/store_db/libmdbx.rs +++ b/crates/storage/store_db/libmdbx.rs @@ -70,6 +70,15 @@ impl Store { txn.commit().map_err(StoreError::LibmdbxError) } + fn write_with_txn( + txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + key: T::Key, + value: T::Value, + ) -> Result<(), StoreError> { + txn.upsert::(key, value) + .map_err(StoreError::LibmdbxError) + } + fn write_batch_with_txn( txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, key_values: impl Iterator, @@ -117,6 +126,30 @@ impl Store { Ok(self.read::(number)?.map(|a| a.to())) } + fn add_block_header_with_txn( + block_hash: BlockHash, + block_header: BlockHeader, + write_txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + ) -> Result<(), StoreError> { + Self::write_with_txn::(write_txn, block_hash.into(), block_header.into()) + } + + fn add_block_number_with_txn( + block_hash: BlockHash, + block_number: BlockNumber, + write_txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + ) -> Result<(), StoreError> { + Self::write_with_txn::(write_txn, block_hash.into(), block_number) + } + + fn add_block_body_with_txn( + block_hash: BlockHash, + block_body: BlockBody, + write_txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, + ) -> Result<(), StoreError> { + Self::write_with_txn::(write_txn, block_hash.into(), block_body.into()) + } + fn add_transaction_locations_with_txn( locations: Vec<(H256, BlockNumber, BlockHash, Index)>, write_txn: &mut libmdbx::orm::Transaction<'_, libmdbx::RW>, @@ -137,6 +170,17 @@ impl Store { ) -> Result<(), StoreError> { self.write_with_closure(move |txn| Self::add_transaction_locations_with_txn(locations, txn)) } + + pub fn add_block_body_and_header(&self, block: Block) -> Result<(), StoreError> { + let header = block.header; + let number = header.number; + let hash = header.compute_block_hash(); + self.write_with_closure(move |txn| { + Self::add_block_body_with_txn(hash, block.body, txn)?; + Self::add_block_header_with_txn(hash, header, txn)?; + Self::add_block_number_with_txn(hash, number, txn) + }) + } } impl StoreEngine for Store { From 569e8456c2e4dc4d70a959706a8efd6a4a2d8e24 Mon Sep 17 00:00:00 2001 From: darshan <8559992+lakshya-sky@users.noreply.github.com> Date: Thu, 27 Feb 2025 14:27:05 +0530 Subject: [PATCH 3/3] use closure fn for storing block stuff --- crates/storage/store_db/libmdbx.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/storage/store_db/libmdbx.rs b/crates/storage/store_db/libmdbx.rs index 98c079a81..300509339 100644 --- a/crates/storage/store_db/libmdbx.rs +++ b/crates/storage/store_db/libmdbx.rs @@ -168,14 +168,14 @@ impl Store { &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> Result<(), StoreError> { - self.write_with_closure(move |txn| Self::add_transaction_locations_with_txn(locations, txn)) + self.write_with_closure(|txn| Self::add_transaction_locations_with_txn(locations, txn)) } pub fn add_block_body_and_header(&self, block: Block) -> Result<(), StoreError> { let header = block.header; let number = header.number; let hash = header.compute_block_hash(); - self.write_with_closure(move |txn| { + self.write_with_closure(|txn| { Self::add_block_body_with_txn(hash, block.body, txn)?; Self::add_block_header_with_txn(hash, header, txn)?; Self::add_block_number_with_txn(hash, number, txn) @@ -189,7 +189,9 @@ impl StoreEngine for Store { block_hash: BlockHash, block_header: BlockHeader, ) -> Result<(), StoreError> { - self.write::(block_hash.into(), block_header.into()) + self.write_with_closure(|txn| { + Self::add_block_header_with_txn(block_hash, block_header, txn) + }) } fn add_block_headers( @@ -220,7 +222,7 @@ impl StoreEngine for Store { block_hash: BlockHash, block_body: BlockBody, ) -> Result<(), StoreError> { - self.write::(block_hash.into(), block_body.into()) + self.write_with_closure(|txn| Self::add_block_body_with_txn(block_hash, block_body, txn)) } fn get_block_body(&self, block_number: BlockNumber) -> Result, StoreError> { @@ -250,7 +252,9 @@ impl StoreEngine for Store { block_hash: BlockHash, block_number: BlockNumber, ) -> Result<(), StoreError> { - self.write::(block_hash.into(), block_number) + self.write_with_closure(|txn| { + Self::add_block_number_with_txn(block_hash, block_number, txn) + }) } fn get_block_number(&self, block_hash: BlockHash) -> Result, StoreError> {