From 1bb6afebfd99e3a1b13fe5968f4f99eafef1174e Mon Sep 17 00:00:00 2001 From: Denis Carriere Date: Fri, 21 Feb 2025 14:19:55 -0500 Subject: [PATCH] Add Keccak Preimages --- blocks/evm/README.md | 3 +- blocks/evm/src/events.rs | 2 ++ blocks/evm/src/keccak_preimages.rs | 44 ++++++++++++++++++++++++++++ blocks/evm/src/lib.rs | 1 + blocks/evm/src/pb/pinax.evm.v1.rs | 47 ++++++++++++++++++++++++++---- blocks/evm/src/storage_changes.rs | 18 +++++++++--- blocks/evm/substreams.yaml | 2 +- proto/evm.proto | 32 ++++++++++++++++---- 8 files changed, 133 insertions(+), 16 deletions(-) create mode 100644 blocks/evm/src/keccak_preimages.rs diff --git a/blocks/evm/README.md b/blocks/evm/README.md index 6e9a1bf..a1f0d0f 100644 --- a/blocks/evm/README.md +++ b/blocks/evm/README.md @@ -18,6 +18,7 @@ - [x] **Account Creation** - [x] **Gas Changes** - [x] **Nonce Changes** + - [x] **Keccak Preimages** ## Graph @@ -37,5 +38,5 @@ Kind: map Input: source: sf.substreams.v1.Clock Input: source: sf.ethereum.type.v2.Block Output Type: proto:pinax.evm.v1.Events -Hash: 700ee7417a6e24ceb49db71e62d030aed84b017d +Hash: 161abf4cee38d9b2e00fd825ad7e5a7df32b8599 ``` diff --git a/blocks/evm/src/events.rs b/blocks/evm/src/events.rs index 97e8440..37f1ba5 100644 --- a/blocks/evm/src/events.rs +++ b/blocks/evm/src/events.rs @@ -9,6 +9,7 @@ use crate::balance_changes::collect_balance_changes; use crate::blocks::collect_block; use crate::code_changes::collect_code_changes; use crate::gas_changes::collect_gas_changes; +use crate::keccak_preimages::collect_keccak_preimages; use crate::logs::collect_logs; use crate::nonce_changes::collect_nonce_changes; use crate::pb::pinax::evm::v1::Events; @@ -31,5 +32,6 @@ pub fn map_events(clock: Clock, block: Block) -> Result { account_creations: collect_account_creations(&block, ×tamp), nonce_changes: collect_nonce_changes(&block, ×tamp), gas_changes: collect_gas_changes(&block, ×tamp), + keccak_preimages: collect_keccak_preimages(&block, ×tamp), }) } diff --git a/blocks/evm/src/keccak_preimages.rs b/blocks/evm/src/keccak_preimages.rs new file mode 100644 index 0000000..5a359a2 --- /dev/null +++ b/blocks/evm/src/keccak_preimages.rs @@ -0,0 +1,44 @@ +use std::collections::HashMap; + +use common::structs::BlockTimestamp; +use substreams_ethereum::pb::eth::v2::Block; + +use crate::pb::pinax::evm::v1::KeccakPreimage; + +// https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L647 +// DetailLevel: EXTENDED +pub fn collect_keccak_preimages(block: &Block, timestamp: &BlockTimestamp) -> Vec { + let mut keccak_preimages_map = HashMap::new(); + let mut keccak_preimages = Vec::new(); + + // Collect storage changes from system calls + for call in &block.system_calls { + for (hash, preimage) in &call.keccak_preimages { + keccak_preimages_map.insert(hash, preimage); + } + } + + // Collect storage changes from transaction traces + for transaction in &block.transaction_traces { + for call in &transaction.calls { + for (hash, preimage) in &call.keccak_preimages { + keccak_preimages_map.insert(hash, preimage); + } + } + } + + for (hash, preimage) in keccak_preimages_map { + keccak_preimages.push(KeccakPreimage { + // block + block_time: timestamp.time.to_string(), + block_number: timestamp.number, + block_hash: timestamp.hash.clone(), + block_date: timestamp.date.clone(), + + // keccak preimage + hash: hash.to_string(), + preimage: preimage.to_string(), + }); + } + keccak_preimages +} diff --git a/blocks/evm/src/lib.rs b/blocks/evm/src/lib.rs index a35207e..f54d7e2 100644 --- a/blocks/evm/src/lib.rs +++ b/blocks/evm/src/lib.rs @@ -10,3 +10,4 @@ mod pb; mod storage_changes; mod traces; mod transactions; +mod keccak_preimages; \ No newline at end of file diff --git a/blocks/evm/src/pb/pinax.evm.v1.rs b/blocks/evm/src/pb/pinax.evm.v1.rs index 198315c..2ee149f 100644 --- a/blocks/evm/src/pb/pinax.evm.v1.rs +++ b/blocks/evm/src/pb/pinax.evm.v1.rs @@ -25,6 +25,8 @@ pub struct Events { pub nonce_changes: ::prost::alloc::vec::Vec, #[prost(message, repeated, tag="10")] pub gas_changes: ::prost::alloc::vec::Vec, + #[prost(message, repeated, tag="11")] + pub keccak_preimages: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -387,19 +389,54 @@ pub struct StorageChange { /// -- transaction -- #[prost(string, optional, tag="5")] pub tx_hash: ::core::option::Option<::prost::alloc::string::String>, + /// -- call -- + #[prost(uint32, tag="10")] + pub call_index: u32, + #[prost(uint32, tag="11")] + pub call_parent_index: u32, + #[prost(uint64, tag="12")] + pub call_begin_ordinal: u64, + #[prost(uint64, tag="13")] + pub call_end_ordinal: u64, + #[prost(uint32, tag="14")] + pub call_depth: u32, /// -- storage change -- /// /// block global ordinal - #[prost(uint64, tag="6")] + #[prost(uint64, tag="20")] pub ordinal: u64, - #[prost(string, tag="7")] + #[prost(string, tag="21")] pub address: ::prost::alloc::string::String, - #[prost(string, tag="8")] + #[prost(string, tag="22")] pub key: ::prost::alloc::string::String, - #[prost(string, tag="9")] + #[prost(string, tag="23")] pub new_value: ::prost::alloc::string::String, - #[prost(string, tag="10")] + #[prost(string, tag="24")] + pub new_value_number: ::prost::alloc::string::String, + #[prost(string, tag="25")] pub old_value: ::prost::alloc::string::String, + #[prost(string, tag="26")] + pub old_value_number: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct KeccakPreimage { + /// -- block -- + /// + /// TIMESTAMP + #[prost(string, tag="1")] + pub block_time: ::prost::alloc::string::String, + #[prost(uint64, tag="2")] + pub block_number: u64, + #[prost(string, tag="3")] + pub block_hash: ::prost::alloc::string::String, + #[prost(string, tag="4")] + pub block_date: ::prost::alloc::string::String, + /// -- keccak preimages -- + #[prost(string, tag="5")] + pub hash: ::prost::alloc::string::String, + #[prost(string, tag="6")] + pub preimage: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] diff --git a/blocks/evm/src/storage_changes.rs b/blocks/evm/src/storage_changes.rs index 0b861c7..b92ba75 100644 --- a/blocks/evm/src/storage_changes.rs +++ b/blocks/evm/src/storage_changes.rs @@ -1,7 +1,8 @@ use common::structs::BlockTimestamp; use common::utils::bytes_to_hex; -use substreams_ethereum::pb::eth::v2::{Block, StorageChange, TransactionTrace}; +use substreams::scalar::BigInt; +use substreams_ethereum::pb::eth::v2::{Block, Call, StorageChange, TransactionTrace}; use crate::pb::pinax::evm::v1::StorageChange as StorageChangeEvent; @@ -13,7 +14,7 @@ pub fn collect_storage_changes(block: &Block, timestamp: &BlockTimestamp) -> Vec // Collect storage changes from system calls for call in &block.system_calls { for storage_change in &call.storage_changes { - storage_changes.push(parse_storage_change(storage_change, &TransactionTrace::default(), timestamp)); + storage_changes.push(parse_storage_change(storage_change, &TransactionTrace::default(), call, timestamp)); } } @@ -21,7 +22,7 @@ pub fn collect_storage_changes(block: &Block, timestamp: &BlockTimestamp) -> Vec for transaction in &block.transaction_traces { for call in &transaction.calls { for storage_change in &call.storage_changes { - storage_changes.push(parse_storage_change(storage_change, transaction, timestamp)); + storage_changes.push(parse_storage_change(storage_change, transaction, call, timestamp)); } } } @@ -29,7 +30,7 @@ pub fn collect_storage_changes(block: &Block, timestamp: &BlockTimestamp) -> Vec storage_changes } -pub fn parse_storage_change(storage_change: &StorageChange, transaction: &TransactionTrace, timestamp: &BlockTimestamp) -> StorageChangeEvent { +pub fn parse_storage_change(storage_change: &StorageChange, transaction: &TransactionTrace, call: &Call, timestamp: &BlockTimestamp) -> StorageChangeEvent { StorageChangeEvent { // block block_time: timestamp.time.to_string(), @@ -40,11 +41,20 @@ pub fn parse_storage_change(storage_change: &StorageChange, transaction: &Transa // transaction tx_hash: Some(bytes_to_hex(&transaction.hash)), + // call + call_index: call.index, + call_parent_index: call.parent_index, + call_begin_ordinal: call.begin_ordinal, + call_end_ordinal: call.end_ordinal, + call_depth: call.depth, + // storage changes address: bytes_to_hex(&storage_change.address), key: bytes_to_hex(&storage_change.key), new_value: bytes_to_hex(&storage_change.new_value), + new_value_number: BigInt::from_unsigned_bytes_be(&storage_change.new_value).to_string(), old_value: bytes_to_hex(&storage_change.old_value), + old_value_number: BigInt::from_unsigned_bytes_be(&storage_change.old_value).to_string(), ordinal: storage_change.ordinal, } } diff --git a/blocks/evm/substreams.yaml b/blocks/evm/substreams.yaml index 1d82c23..a3fdb98 100644 --- a/blocks/evm/substreams.yaml +++ b/blocks/evm/substreams.yaml @@ -1,7 +1,7 @@ specVersion: v0.1.0 package: name: raw_blocks_evm - version: v1.0.0 + version: v1.1.0 url: https://github.com/pinax-network/substreams-raw-blocks image: logo.png diff --git a/proto/evm.proto b/proto/evm.proto index 7ec706d..f402af5 100644 --- a/proto/evm.proto +++ b/proto/evm.proto @@ -15,6 +15,7 @@ message Events { repeated AccountCreation account_creations = 8; repeated NonceChange nonce_changes = 9; repeated GasChange gas_changes = 10; + repeated KeccakPreimage keccak_preimages = 11; } message Block { @@ -210,12 +211,33 @@ message StorageChange { // -- transaction -- optional string tx_hash = 5; + // -- call -- + uint32 call_index = 10; + uint32 call_parent_index = 11; + uint64 call_begin_ordinal = 12; + uint64 call_end_ordinal = 13; + uint32 call_depth = 14; + // -- storage change -- - uint64 ordinal = 6; // block global ordinal - string address = 7; - string key = 8; - string new_value = 9; - string old_value = 10; + uint64 ordinal = 20; // block global ordinal + string address = 21; + string key = 22; + string new_value = 23; + string new_value_number = 24; + string old_value = 25; + string old_value_number = 26; +} + +message KeccakPreimage { + // -- block -- + string block_time = 1; // TIMESTAMP + uint64 block_number = 2; + string block_hash = 3; + string block_date = 4; + + // -- keccak preimages -- + string hash = 5; + string preimage = 6; } message CodeChange {