Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

offchain DB MMR prunning functionality and tests #269

Closed
wants to merge 3 commits into from
Closed

offchain DB MMR prunning functionality and tests #269

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2cc7642
initial work
MrishoLukamba Aug 2, 2024
3e0b3aa
prune on the earliest peak
MrishoLukamba Aug 2, 2024
2a95df9
prune on the earliest peak
MrishoLukamba Aug 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push:
branches:
- main
pull_request_target:
pull_request:
types: [opened, synchronize]
# pull_request:
# branches:
Expand Down
2 changes: 2 additions & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 5 additions & 10 deletions evm/abi/src/generated/evm_host.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2712,8 +2712,7 @@ pub mod evm_host {
///Gets the contract's `GetRequestHandled` event
pub fn get_request_handled_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, GetRequestHandledFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, GetRequestHandledFilter> {
self.0.event()
}
///Gets the contract's `GetRequestTimeoutHandled` event
Expand All @@ -2735,8 +2734,7 @@ pub mod evm_host {
///Gets the contract's `HostParamsUpdated` event
pub fn host_params_updated_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, HostParamsUpdatedFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, HostParamsUpdatedFilter> {
self.0.event()
}
///Gets the contract's `HostWithdrawal` event
Expand All @@ -2754,8 +2752,7 @@ pub mod evm_host {
///Gets the contract's `PostRequestHandled` event
pub fn post_request_handled_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostRequestHandledFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostRequestHandledFilter> {
self.0.event()
}
///Gets the contract's `PostRequestTimeoutHandled` event
Expand All @@ -2771,15 +2768,13 @@ pub mod evm_host {
///Gets the contract's `PostResponseEvent` event
pub fn post_response_event_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostResponseEventFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostResponseEventFilter> {
self.0.event()
}
///Gets the contract's `PostResponseFunded` event
pub fn post_response_funded_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostResponseFundedFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, PostResponseFundedFilter> {
self.0.event()
}
///Gets the contract's `PostResponseHandled` event
Expand Down
6 changes: 2 additions & 4 deletions evm/abi/src/generated/ping_module.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -699,15 +699,13 @@ pub mod ping_module {
///Gets the contract's `GetTimeoutReceived` event
pub fn get_timeout_received_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, GetTimeoutReceivedFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, GetTimeoutReceivedFilter> {
self.0.event()
}
///Gets the contract's `MessageDispatched` event
pub fn message_dispatched_filter(
&self,
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, MessageDispatchedFilter>
{
) -> ::ethers::contract::builders::Event<::std::sync::Arc<M>, M, MessageDispatchedFilter> {
self.0.event()
}
///Gets the contract's `PostReceived` event
Expand Down
21 changes: 7 additions & 14 deletions modules/consensus/beefy/prover/src/runtime/paseo.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -526,8 +526,7 @@ pub mod api {
pub fn pending_rewards(
&self,
who: ::subxt::utils::AccountId32,
) -> ::subxt::runtime_api::Payload<types::PendingRewards, ::core::primitive::u128>
{
) -> ::subxt::runtime_api::Payload<types::PendingRewards, ::core::primitive::u128> {
::subxt::runtime_api::Payload::new_static(
"NominationPoolsApi",
"pending_rewards",
Expand Down Expand Up @@ -2247,8 +2246,7 @@ pub mod api {
#[doc = " Get current GRANDPA authority set id."]
pub fn current_set_id(
&self,
) -> ::subxt::runtime_api::Payload<types::CurrentSetId, ::core::primitive::u64>
{
) -> ::subxt::runtime_api::Payload<types::CurrentSetId, ::core::primitive::u64> {
::subxt::runtime_api::Payload::new_static(
"GrandpaApi",
"current_set_id",
Expand Down Expand Up @@ -2685,8 +2683,7 @@ pub mod api {
pub fn account_nonce(
&self,
account: ::subxt::utils::AccountId32,
) -> ::subxt::runtime_api::Payload<types::AccountNonce, ::core::primitive::u32>
{
) -> ::subxt::runtime_api::Payload<types::AccountNonce, ::core::primitive::u32> {
::subxt::runtime_api::Payload::new_static(
"AccountNonceApi",
"account_nonce",
Expand Down Expand Up @@ -4398,8 +4395,7 @@ pub mod api {
#[doc = " The maximum length of a block (in bytes)."]
pub fn block_length(
&self,
) -> ::subxt::constants::Address<runtime_types::frame_system::limits::BlockLength>
{
) -> ::subxt::constants::Address<runtime_types::frame_system::limits::BlockLength> {
::subxt::constants::Address::new_static(
"System",
"BlockLength",
Expand Down Expand Up @@ -12689,8 +12685,7 @@ pub mod api {
#[doc = " Maximum amount of funds that should be placed in a deposit for making a proposal."]
pub fn proposal_bond_maximum(
&self,
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>>
{
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>> {
::subxt::constants::Address::new_static(
"Treasury",
"ProposalBondMaximum",
Expand Down Expand Up @@ -18654,8 +18649,7 @@ pub mod api {
#[doc = " Maximum amount of funds that should be placed in a deposit for making a proposal."]
pub fn curator_deposit_max(
&self,
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>>
{
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>> {
::subxt::constants::Address::new_static(
"Bounties",
"CuratorDepositMax",
Expand All @@ -18670,8 +18664,7 @@ pub mod api {
#[doc = " Minimum amount of funds that should be placed in a deposit for making a proposal."]
pub fn curator_deposit_min(
&self,
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>>
{
) -> ::subxt::constants::Address<::core::option::Option<::core::primitive::u128>> {
::subxt::constants::Address::new_static(
"Bounties",
"CuratorDepositMin",
Expand Down
4 changes: 4 additions & 0 deletions modules/trees/mmr/pallet/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ mmr-primitives = { workspace = true }
sp-runtime = { workspace = true }
sp-std = { workspace = true }
merkle-mountain-range = { workspace = true }
itertools = { version = "0.10.3", default-features = false }
pallet-ismp = { workspace = true }
ismp = { workspace = true }

[dev-dependencies]
array-bytes = "6.1"
Expand All @@ -50,6 +53,7 @@ std = [
"sp-std/std",
"merkle-mountain-range/std",
"serde/default",
"itertools/use_std"
]
runtime-benchmarks = [
"frame-benchmarking/runtime-benchmarks",
Expand Down
101 changes: 95 additions & 6 deletions modules/trees/mmr/pallet/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,23 +56,32 @@
//! NOTE This pallet is experimental and not proven to work in production.
#![cfg_attr(not(feature = "std"), no_std)]

use codec::Decode;
use core::marker::PhantomData;
use frame_system::pallet_prelude::{BlockNumberFor, HeaderFor};
use itertools::Itertools;
use log;
use merkle_mountain_range::MMRStore;
use log::trace;
use merkle_mountain_range::{helper::pos_height_in_tree, MMRStore};
use sp_core::H256;
use sp_core::offchain::StorageKind;

use sp_runtime::traits::{self, One};
use sp_std::prelude::*;

use mmr_primitives::{DataOrHash, LeafMetadata, MerkleMountainRangeTree};
use ismp::{
messaging::{hash_request, Keccak256},
router::Request,
};
use mmr_primitives::{DataOrHash, FullLeaf, LeafMetadata, MerkleMountainRangeTree};
pub use pallet::*;
use sp_mmr_primitives::mmr_lib::leaf_index_to_pos;
pub use sp_mmr_primitives::{
self as primitives, utils::NodesUtils, Error, LeafDataProvider, LeafIndex, NodeIndex,
};
use sp_mmr_primitives::mmr_lib::leaf_index_to_pos;

pub use mmr::storage::{OffchainStorage, Storage};
use pallet_ismp::NoOpMmrTree;

pub mod mmr;

Expand All @@ -99,7 +108,7 @@ pub mod pallet {

/// This pallet's configuration trait
#[pallet::config]
pub trait Config<I: 'static = ()>: frame_system::Config {
pub trait Config<I: 'static = ()>: frame_system::Config + pallet_ismp::Config {
/// Prefix for elements stored in the Off-chain DB via Indexing API.
///
/// Each node of the MMR is inserted both on-chain and off-chain via Indexing API.
Expand Down Expand Up @@ -127,6 +136,9 @@ pub mod pallet {

/// A type that returns a hash unique to every block as a fork identifer for offchain keys
type ForkIdentifierProvider: ForkIdentifier<Self>;

/// Leaves count to prune
const LEAF_COUNT_THRESHOLD: u64;
}

/// Latest MMR Root hash.
Expand Down Expand Up @@ -163,14 +175,21 @@ pub mod pallet {
// Set the initial height at which leaves were pushed to the offchain db for the offchain
// mmr gadget. Since this is in on_initialize, then the leaves were set in a previous block.
#[pallet::hooks]
impl<T: Config<I>, I: 'static> Hooks<BlockNumberFor<T>> for Pallet<T, I> {
impl<T: Config<I>, I: 'static> Hooks<BlockNumberFor<T>> for Pallet<T, I>
where
HashOf<T, I>: Into<H256>,
{
fn on_initialize(_n: BlockNumberFor<T>) -> Weight {
if NumberOfLeaves::<T, I>::get() > 0 && InitialHeight::<T, I>::get().is_none() {
InitialHeight::<T, I>::put(frame_system::Pallet::<T>::block_number() - One::one())
}

Default::default()
}
fn on_idle(_n: BlockNumberFor<T>, _remaining_weight: Weight) -> Weight {
Self::prune_mmr_leaves().unwrap(); // It should not panic
Default::default()
}
}
}

Expand Down Expand Up @@ -264,8 +283,77 @@ where
})
.map_err(|_| Error::LeafNotFound)
}
}

// fetch the peaks and under each peak see if latest leaves (i.e right most leaf (leafIndex) is
// still valid, meaning has not timedout or processed. if its invalid then prune all leaves and
// inner under the peak. for now only prune the earliest peak
fn prune_mmr_leaves() -> Result<(), Error> {
if Self::leaf_count() < T::LEAF_COUNT_THRESHOLD {
Ok(())?
}

let peaks_indexs = Nodes::<T, I>::iter()
.sorted_by_key(|(k, _)| *k)
.map(|(k, _v)| k)
.collect::<Vec<NodeIndex>>();
// if there is only 1 peak meaning the tree has no of leaves 2^n we are pruning inner nodes
// i.e the left most inner node on the (h-1) layer, where h = height of the tree
// if there is more than 1 peak, then prune all the nodes on the first peak
if peaks_indexs.len() == 1 {
todo!()
} else {
if let Some(peak_index) = peaks_indexs.iter().next() {
// get the last leaf under the peak and check
let last_leaf_index = *peak_index - pos_height_in_tree(*peak_index) as u64;
if let Some(leaf_type) = Self::get_leaf(last_leaf_index)? {
let last_leaf_to_delete = {
let encoded_inner_leaf = leaf_type.preimage();
let leaf_request: ismp::router::Request =
Decode::decode(&mut &encoded_inner_leaf[..])
.map_err(|_| Error::LeafNotFound)?;

let claimed =
pallet_ismp::child_trie::RequestCommitments::<T>::get(hash_request::<
pallet_ismp::Pallet<T>,
>(&leaf_request))
.ok_or(Error::LeafNotFound)?
.claimed;

match leaf_request {
Request::Post(post) => {
// check if it has timedout and if fees has been claimed by this
// request
let _timeout = post.timeout_timestamp;
claimed
},
Request::Get(ref get) => {
// check if it has timedout and if fees has been claimed by this
// request
let _timeout = get.timeout_timestamp;
claimed
},
}
};

// if we can delete the last leaf we can delete all nodes under the peak
if last_leaf_to_delete {
for node_index in 0..*peak_index {
let leaf = Self::get_leaf(node_index)?.ok_or(Error::LeafNotFound)?;
let commitment = pallet_ismp::Pallet::<T>::keccak256(&leaf.preimage()[..]);
// delete the node
let offchain_key = NoOpMmrTree::<T>::offchain_key(commitment);
sp_io::offchain::local_storage_clear(StorageKind::PERSISTENT, &offchain_key)
}
}else{
trace!(target: "mmr:pruning","No nodes to prune")
}
}
}
}

Ok(())
}
}
/// Stateless MMR proof verification for batch of leaves.
///
/// This function can be used to verify received MMR [primitives::Proof] (`proof`)
Expand Down Expand Up @@ -361,3 +449,4 @@ impl<T: Config<I>, I: 'static> Pallet<T, I> {
}
}
}

10 changes: 9 additions & 1 deletion modules/trees/mmr/primitives/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use core::marker::PhantomData;
use merkle_mountain_range::helper::{get_peaks, parent_offset, pos_height_in_tree, sibling_offset};
use sp_core::H256;
use sp_mmr_primitives as primitives;
use sp_mmr_primitives::NodeIndex;
use sp_mmr_primitives::{Error, NodeIndex};
use sp_runtime::{scale_info, traits, RuntimeDebug};
use sp_std::fmt;

Expand Down Expand Up @@ -63,6 +63,10 @@ pub trait MerkleMountainRangeTree {

/// Given the leaf position, it should return the leaf from the mmr store
fn get_leaf(pos: NodeIndex) -> Result<Option<Self::Leaf>, primitives::Error>;

/// Prune the leaves of the mmr stored offchain/onchain based on specific implementation while
/// maintaining MMR integrity (i.e leaving the peaks of the pruned leaves)
fn prune_mmr_leaves() -> Result<(), primitives::Error>;
}

/// NoOp tree can be used as a drop in replacement for when the underlying mmr tree is unneeded.
Expand Down Expand Up @@ -105,6 +109,10 @@ impl<T: FullLeaf, H: ismp::messaging::Keccak256> MerkleMountainRangeTree for NoO
fn get_leaf(_pos: NodeIndex) -> Result<Option<Self::Leaf>, primitives::Error> {
Ok(None)
}

fn prune_mmr_leaves() -> Result<(), Error> {
Ok(())
}
}

/// A full leaf content stored in the offchain-db.
Expand Down
9 changes: 3 additions & 6 deletions modules/utils/subxt/src/gargantua.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,7 @@ pub mod api {
&self,
included_hash: ::subxt::utils::H256,
slot: runtime_types::sp_consensus_slots::Slot,
) -> ::subxt::runtime_api::Payload<types::CanBuildUpon, ::core::primitive::bool>
{
) -> ::subxt::runtime_api::Payload<types::CanBuildUpon, ::core::primitive::bool> {
::subxt::runtime_api::Payload::new_static(
"AuraUnincludedSegmentApi",
"can_build_upon",
Expand Down Expand Up @@ -854,8 +853,7 @@ pub mod api {
pub fn account_nonce(
&self,
account: ::subxt::utils::AccountId32,
) -> ::subxt::runtime_api::Payload<types::AccountNonce, ::core::primitive::u32>
{
) -> ::subxt::runtime_api::Payload<types::AccountNonce, ::core::primitive::u32> {
::subxt::runtime_api::Payload::new_static(
"AccountNonceApi",
"account_nonce",
Expand Down Expand Up @@ -3253,8 +3251,7 @@ pub mod api {
#[doc = " The maximum length of a block (in bytes)."]
pub fn block_length(
&self,
) -> ::subxt::constants::Address<runtime_types::frame_system::limits::BlockLength>
{
) -> ::subxt::constants::Address<runtime_types::frame_system::limits::BlockLength> {
::subxt::constants::Address::new_static(
"System",
"BlockLength",
Expand Down
Loading