Add Keccak Preimages

blocks/evm/README.md

@@ -18,6 +18,7 @@
   - [x] **Account Creation**
   - [x] **Gas Changes**
   - [x] **Nonce Changes**
+  - [x] **Keccak Preimages**
 
 ## Graph
 
@@ -37,5 +38,5 @@ Kind: map
 Input: source: sf.substreams.v1.Clock
 Input: source: sf.ethereum.type.v2.Block
 Output Type: proto:pinax.evm.v1.Events
-Hash: 700ee7417a6e24ceb49db71e62d030aed84b017d
+Hash: 161abf4cee38d9b2e00fd825ad7e5a7df32b8599
 ```

blocks/evm/src/events.rs

@@ -9,6 +9,7 @@ use crate::balance_changes::collect_balance_changes;
 use crate::blocks::collect_block;
 use crate::code_changes::collect_code_changes;
 use crate::gas_changes::collect_gas_changes;
+use crate::keccak_preimages::collect_keccak_preimages;
 use crate::logs::collect_logs;
 use crate::nonce_changes::collect_nonce_changes;
 use crate::pb::pinax::evm::v1::Events;
@@ -31,5 +32,6 @@ pub fn map_events(clock: Clock, block: Block) -> Result<Events, Error> {
         account_creations: collect_account_creations(&block, &timestamp),
         nonce_changes: collect_nonce_changes(&block, &timestamp),
         gas_changes: collect_gas_changes(&block, &timestamp),
+        keccak_preimages: collect_keccak_preimages(&block, &timestamp),
     })
 }

blocks/evm/src/keccak_preimages.rs

@@ -0,0 +1,44 @@
+use std::collections::HashMap;
+
+use common::structs::BlockTimestamp;
+use substreams_ethereum::pb::eth::v2::Block;
+
+use crate::pb::pinax::evm::v1::KeccakPreimage;
+
+// https://github.com/streamingfast/firehose-ethereum/blob/1bcb32a8eb3e43347972b6b5c9b1fcc4a08c751e/proto/sf/ethereum/type/v2/type.proto#L647
+// DetailLevel: EXTENDED
+pub fn collect_keccak_preimages(block: &Block, timestamp: &BlockTimestamp) -> Vec<KeccakPreimage> {
+    let mut keccak_preimages_map = HashMap::new();
+    let mut keccak_preimages = Vec::new();
+
+    // Collect storage changes from system calls
+    for call in &block.system_calls {
+        for (hash, preimage) in &call.keccak_preimages {
+            keccak_preimages_map.insert(hash, preimage);
+        }
+    }
+
+    // Collect storage changes from transaction traces
+    for transaction in &block.transaction_traces {
+        for call in &transaction.calls {
+            for (hash, preimage) in &call.keccak_preimages {
+                keccak_preimages_map.insert(hash, preimage);
+            }
+        }
+    }
+
+    for (hash, preimage) in keccak_preimages_map {
+        keccak_preimages.push(KeccakPreimage {
+            // block
+            block_time: timestamp.time.to_string(),
+            block_number: timestamp.number,
+            block_hash: timestamp.hash.clone(),
+            block_date: timestamp.date.clone(),
+
+            // keccak preimage
+            hash: hash.to_string(),
+            preimage: preimage.to_string(),
+        });
+    }
+    keccak_preimages
+}

blocks/evm/src/lib.rs

@@ -10,3 +10,4 @@ mod pb;
 mod storage_changes;
 mod traces;
 mod transactions;
+mod keccak_preimages;

blocks/evm/src/pb/pinax.evm.v1.rs

@@ -25,6 +25,8 @@ pub struct Events {
     pub nonce_changes: ::prost::alloc::vec::Vec<NonceChange>,
     #[prost(message, repeated, tag="10")]
     pub gas_changes: ::prost::alloc::vec::Vec<GasChange>,
+    #[prost(message, repeated, tag="11")]
+    pub keccak_preimages: ::prost::alloc::vec::Vec<KeccakPreimage>,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]
@@ -387,19 +389,54 @@ pub struct StorageChange {
     /// -- transaction --
     #[prost(string, optional, tag="5")]
     pub tx_hash: ::core::option::Option<::prost::alloc::string::String>,
+    /// -- call --
+    #[prost(uint32, tag="10")]
+    pub call_index: u32,
+    #[prost(uint32, tag="11")]
+    pub call_parent_index: u32,
+    #[prost(uint64, tag="12")]
+    pub call_begin_ordinal: u64,
+    #[prost(uint64, tag="13")]
+    pub call_end_ordinal: u64,
+    #[prost(uint32, tag="14")]
+    pub call_depth: u32,
     /// -- storage change --
     ///
     /// block global ordinal
-    #[prost(uint64, tag="6")]
+    #[prost(uint64, tag="20")]
     pub ordinal: u64,
-    #[prost(string, tag="7")]
+    #[prost(string, tag="21")]
     pub address: ::prost::alloc::string::String,
-    #[prost(string, tag="8")]
+    #[prost(string, tag="22")]
     pub key: ::prost::alloc::string::String,
-    #[prost(string, tag="9")]
+    #[prost(string, tag="23")]
     pub new_value: ::prost::alloc::string::String,
-    #[prost(string, tag="10")]
+    #[prost(string, tag="24")]
+    pub new_value_number: ::prost::alloc::string::String,
+    #[prost(string, tag="25")]
     pub old_value: ::prost::alloc::string::String,
+    #[prost(string, tag="26")]
+    pub old_value_number: ::prost::alloc::string::String,
+}
+#[allow(clippy::derive_partial_eq_without_eq)]
+#[derive(Clone, PartialEq, ::prost::Message)]
+pub struct KeccakPreimage {
+    /// -- block --
+    ///
+    /// TIMESTAMP
+    #[prost(string, tag="1")]
+    pub block_time: ::prost::alloc::string::String,
+    #[prost(uint64, tag="2")]
+    pub block_number: u64,
+    #[prost(string, tag="3")]
+    pub block_hash: ::prost::alloc::string::String,
+    #[prost(string, tag="4")]
+    pub block_date: ::prost::alloc::string::String,
+    /// -- keccak preimages --
+    #[prost(string, tag="5")]
+    pub hash: ::prost::alloc::string::String,
+    #[prost(string, tag="6")]
+    pub preimage: ::prost::alloc::string::String,
 }
 #[allow(clippy::derive_partial_eq_without_eq)]
 #[derive(Clone, PartialEq, ::prost::Message)]

blocks/evm/src/storage_changes.rs

@@ -1,7 +1,8 @@
 use common::structs::BlockTimestamp;
 use common::utils::bytes_to_hex;
 
-use substreams_ethereum::pb::eth::v2::{Block, StorageChange, TransactionTrace};
+use substreams::scalar::BigInt;
+use substreams_ethereum::pb::eth::v2::{Block, Call, StorageChange, TransactionTrace};
 
 use crate::pb::pinax::evm::v1::StorageChange as StorageChangeEvent;
 
@@ -13,23 +14,23 @@ pub fn collect_storage_changes(block: &Block, timestamp: &BlockTimestamp) -> Vec
     // Collect storage changes from system calls
     for call in &block.system_calls {
         for storage_change in &call.storage_changes {
-            storage_changes.push(parse_storage_change(storage_change, &TransactionTrace::default(), timestamp));
+            storage_changes.push(parse_storage_change(storage_change, &TransactionTrace::default(), call, timestamp));
         }
     }
 
     // Collect storage changes from transaction traces
     for transaction in &block.transaction_traces {
         for call in &transaction.calls {
             for storage_change in &call.storage_changes {
-                storage_changes.push(parse_storage_change(storage_change, transaction, timestamp));
+                storage_changes.push(parse_storage_change(storage_change, transaction, call, timestamp));
             }
         }
     }
 
     storage_changes
 }
 
-pub fn parse_storage_change(storage_change: &StorageChange, transaction: &TransactionTrace, timestamp: &BlockTimestamp) -> StorageChangeEvent {
+pub fn parse_storage_change(storage_change: &StorageChange, transaction: &TransactionTrace, call: &Call, timestamp: &BlockTimestamp) -> StorageChangeEvent {
     StorageChangeEvent {
         // block
         block_time: timestamp.time.to_string(),
@@ -40,11 +41,20 @@ pub fn parse_storage_change(storage_change: &StorageChange, transaction: &Transa
         // transaction
         tx_hash: Some(bytes_to_hex(&transaction.hash)),
 
+        // call
+        call_index: call.index,
+        call_parent_index: call.parent_index,
+        call_begin_ordinal: call.begin_ordinal,
+        call_end_ordinal: call.end_ordinal,
+        call_depth: call.depth,
+
         // storage changes
         address: bytes_to_hex(&storage_change.address),
         key: bytes_to_hex(&storage_change.key),
         new_value: bytes_to_hex(&storage_change.new_value),
+        new_value_number: BigInt::from_unsigned_bytes_be(&storage_change.new_value).to_string(),
         old_value: bytes_to_hex(&storage_change.old_value),
+        old_value_number: BigInt::from_unsigned_bytes_be(&storage_change.old_value).to_string(),
         ordinal: storage_change.ordinal,
     }
 }

blocks/evm/substreams.yaml

@@ -1,7 +1,7 @@
 specVersion: v0.1.0
 package:
   name: raw_blocks_evm
-  version: v1.0.0
+  version: v1.1.0
   url: https://github.com/pinax-network/substreams-raw-blocks
   image: logo.png
 

proto/evm.proto

@@ -15,6 +15,7 @@ message Events {
   repeated AccountCreation account_creations = 8;
   repeated NonceChange nonce_changes = 9;
   repeated GasChange gas_changes = 10;
+  repeated KeccakPreimage keccak_preimages = 11;
 }
 
 message Block {
@@ -210,12 +211,33 @@ message StorageChange {
   // -- transaction --
   optional string tx_hash = 5;
 
+  // -- call --
+  uint32 call_index = 10;
+  uint32 call_parent_index = 11;
+  uint64 call_begin_ordinal = 12;
+  uint64 call_end_ordinal = 13;
+  uint32 call_depth = 14;
+
   // -- storage change --
-  uint64 ordinal = 6; // block global ordinal
-  string address = 7;
-  string key = 8;
-  string new_value = 9;
-  string old_value = 10;
+  uint64 ordinal = 20; // block global ordinal
+  string address = 21;
+  string key = 22;
+  string new_value = 23;
+  string new_value_number = 24;
+  string old_value = 25;
+  string old_value_number = 26;
+}
+
+message KeccakPreimage {
+  // -- block --
+  string block_time = 1; // TIMESTAMP
+  uint64 block_number = 2;
+  string block_hash = 3;
+  string block_date = 4;
+
+  // -- keccak preimages --
+  string hash = 5;
+  string preimage = 6;
 }
 
 message CodeChange {